diff --git a/moneymgr_mobile/android/app/build.gradle.kts b/moneymgr_mobile/android/app/build.gradle.kts index 20cd534..ecf9ed6 100644 --- a/moneymgr_mobile/android/app/build.gradle.kts +++ b/moneymgr_mobile/android/app/build.gradle.kts @@ -59,7 +59,7 @@ android { create("development") { dimension = "default" applicationIdSuffix = ".debug" - signingConfig = signingConfigs.getByName("debug") + // signingConfig = signingConfigs.getByName("debug") } create("publish") { dimension = "default" diff --git a/moneymgr_mobile/lib/routes/scan/scan_screen.dart b/moneymgr_mobile/lib/routes/scan/scan_screen.dart index c647cb8..5b1f008 100644 --- a/moneymgr_mobile/lib/routes/scan/scan_screen.dart +++ b/moneymgr_mobile/lib/routes/scan/scan_screen.dart @@ -1,45 +1,23 @@ -import 'dart:io'; import 'dart:typed_data'; import 'package:flutter/material.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:logging/logging.dart'; import 'package:moneymgr_mobile/services/storage/expenses.dart'; +import 'package:moneymgr_mobile/utils/ocr_utils.dart'; +import 'package:moneymgr_mobile/utils/pdf_utils.dart'; import 'package:moneymgr_mobile/widgets/expense_editor.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; -import 'package:scanbot_sdk/scanbot_sdk.dart'; -import 'package:scanbot_sdk/scanbot_sdk_ui_v2.dart' hide IconButton, EdgeInsets; part 'scan_screen.g.dart'; /// Scan a document & return generated PDF as byte file @riverpod -Future _scanDocument(Ref ref) async { - var configuration = DocumentScanningFlow( - appearance: DocumentFlowAppearanceConfiguration( - statusBarMode: StatusBarMode.DARK, - ), - cleanScanningSession: true, - outputSettings: DocumentScannerOutputSettings(pagesScanLimit: 1), - screens: DocumentScannerScreens( - review: ReviewScreenConfiguration(enabled: false), - ), - ); - var documentResult = await ScanbotSdkUiV2.startDocumentScanner(configuration); - - if (documentResult.status != OperationStatus.OK) { - throw Exception("Scanner failed with status ${documentResult.status}"); - } - - // Convert result to PDF - var result = await ScanbotSdk.document.createPDFForDocument( - PDFFromDocumentParams( - documentID: documentResult.data!.uuid, - pdfConfiguration: PdfConfiguration(), - ), - ); - final pdfPath = result.pdfFileUri.replaceFirst("file://", ""); - return File(pdfPath).readAsBytes(); +Future<(Uint8List?, double?)> _scanDocument(Ref ref) async { + final pdf = await scanDocAsPDF(); + final img = await renderPdf(pdfBytes: pdf); + final amount = await extractTotalFromBill(img); + return (pdf, amount); } class ScanScreen extends HookConsumerWidget { @@ -62,12 +40,17 @@ class ScanScreen extends HookConsumerWidget { return Padding( padding: const EdgeInsets.all(8.0), child: switch (scanDocProvider) { - AsyncData(:final value) when value != null => ExpenseEditor( - file: value, + AsyncData(:final value) when value.$1 != null => ExpenseEditor( + file: value.$1!, + initialData: BaseExpenseInfo( + label: null, + cost: value.$2 ?? 0.0, + time: DateTime.now(), + ), onFinished: (expense) async { await expenses.add( info: expense, - fileContent: value, + fileContent: value.$1!, fileMimeType: "application/pdf", ); restartScan(); @@ -76,7 +59,7 @@ class ScanScreen extends HookConsumerWidget { ), // No data - AsyncData(:final value) when value == null => ScanErrorScreen( + AsyncData(:final value) when value.$1 == null => ScanErrorScreen( message: "No document scanned!", onTryAgain: restartScan, ), diff --git a/moneymgr_mobile/lib/utils/ocr_utils.dart b/moneymgr_mobile/lib/utils/ocr_utils.dart new file mode 100644 index 0000000..6a7fd10 --- /dev/null +++ b/moneymgr_mobile/lib/utils/ocr_utils.dart @@ -0,0 +1,44 @@ +import 'dart:math'; +import 'dart:typed_data'; +import 'dart:ui' as ui; + +import 'package:flutter/material.dart'; +import 'package:google_mlkit_text_recognition/google_mlkit_text_recognition.dart'; +import 'package:logging/logging.dart'; + +/// Attempt to extract total amount from invoice image +Future extractTotalFromBill(Uint8List imgBuff) async { + final decodedImage = await decodeImageFromList(imgBuff); + + final byteData = await decodedImage.toByteData( + format: ui.ImageByteFormat.rawRgba, + ); + + final image = InputImage.fromBitmap( + bitmap: byteData!.buffer.asUint8List(), + width: decodedImage.width, + height: decodedImage.height, + ); + + final textRecognizer = TextRecognizer(script: TextRecognitionScript.latin); + final extractionResult = await textRecognizer.processImage(image); + + Logger.root.fine("Expense text: ${extractionResult.text}"); + + // Check for highest amount on invoice + final regexp = RegExp( + r'([0-9]+([ ]*(\\.|,)[ ]*[0-9]{1,2}){0,1})[ \\t\\n]*(EUR|eur|€)', + multiLine: true, + caseSensitive: false, + ); + var highest = 0.0; + for (final match in regexp.allMatches(extractionResult.text)) { + if (match.groupCount == 0) continue; + + // Process only numeric value + final value = (match.group(1) ?? "").replaceAll(",", "."); + highest = max(highest, double.tryParse(value) ?? 0.0); + } + + return highest == 0.0 ? null : highest; +} diff --git a/moneymgr_mobile/lib/utils/pdf_utils.dart b/moneymgr_mobile/lib/utils/pdf_utils.dart index 55da0aa..719d31a 100644 --- a/moneymgr_mobile/lib/utils/pdf_utils.dart +++ b/moneymgr_mobile/lib/utils/pdf_utils.dart @@ -6,14 +6,40 @@ import 'package:flutter/material.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; import 'package:pdf_image_renderer/pdf_image_renderer.dart'; +import 'package:scanbot_sdk/scanbot_sdk.dart'; +import 'package:scanbot_sdk/scanbot_sdk_ui_v2.dart' hide IconButton, EdgeInsets; +/// Scan document as PDF +Future scanDocAsPDF() async { + var configuration = DocumentScanningFlow( + appearance: DocumentFlowAppearanceConfiguration( + statusBarMode: StatusBarMode.DARK, + ), + cleanScanningSession: true, + outputSettings: DocumentScannerOutputSettings(pagesScanLimit: 1), + screens: DocumentScannerScreens( + review: ReviewScreenConfiguration(enabled: false), + ), + ); + var documentResult = await ScanbotSdkUiV2.startDocumentScanner(configuration); + + if (documentResult.status != OperationStatus.OK) { + throw Exception("Scanner failed with status ${documentResult.status}"); + } + + // Convert result to PDF + var result = await ScanbotSdk.document.createPDFForDocument( + PDFFromDocumentParams( + documentID: documentResult.data!.uuid, + pdfConfiguration: PdfConfiguration(), + ), + ); + final pdfPath = result.pdfFileUri.replaceFirst("file://", ""); + return File(pdfPath).readAsBytes(); +} /// Render PDF to image bits -Future renderPdf( - { - String? path, - Uint8List? pdfBytes, - }) async { +Future renderPdf({String? path, Uint8List? pdfBytes}) async { assert(path != null || pdfBytes != null); // Create temporary file if required @@ -62,4 +88,4 @@ Future renderPdf( await File(path).delete(); } } -} \ No newline at end of file +} diff --git a/moneymgr_mobile/pubspec.lock b/moneymgr_mobile/pubspec.lock index 8d5f0e6..d03109d 100644 --- a/moneymgr_mobile/pubspec.lock +++ b/moneymgr_mobile/pubspec.lock @@ -488,6 +488,22 @@ packages: url: "https://pub.dev" source: hosted version: "16.0.0" + google_mlkit_commons: + dependency: transitive + description: + name: google_mlkit_commons + sha256: "8f40fbac10685cad4715d11e6a0d86837d9ad7168684dfcad29610282a88e67a" + url: "https://pub.dev" + source: hosted + version: "0.11.0" + google_mlkit_text_recognition: + dependency: "direct main" + description: + name: google_mlkit_text_recognition + sha256: "96173ad4dd7fd06c660e22ac3f9e9f1798a517fe7e48bee68eeec83853224224" + url: "https://pub.dev" + source: hosted + version: "0.15.0" graphs: dependency: transitive description: diff --git a/moneymgr_mobile/pubspec.yaml b/moneymgr_mobile/pubspec.yaml index e6c5cfa..714ef56 100644 --- a/moneymgr_mobile/pubspec.yaml +++ b/moneymgr_mobile/pubspec.yaml @@ -19,7 +19,7 @@ publish_to: 'none' # Remove this line if you wish to publish to pub.dev version: 1.0.0+1 environment: - sdk: ^3.8.2 + sdk: ^3.8.1 # Dependencies specify other packages that your package needs in order to work. # To automatically upgrade your package dependencies to the latest versions @@ -93,6 +93,9 @@ dependencies: # PDF renderer pdf_image_renderer: ^1.0.1 + # Text extraction + google_mlkit_text_recognition: ^0.15.0 + dev_dependencies: flutter_test: sdk: flutter