Implement local medication crop recognition flow

android-app/app/build.gradle.kts

@@ -12,8 +12,8 @@ android {
         applicationId = "ru.obdai.receipt"
         minSdk = 26
         targetSdk = 35
-        versionCode = 2
+        versionCode = 4
-        versionName = "0.1.1"
+        versionName = "0.1.3"
 
         buildConfigField("String", "RECEIPT_API_TOKEN", "\"\"")
         buildConfigField("String", "RECEIPT_API_URL", "\"https://obdai.ru/receipt\"")
@@ -47,6 +47,7 @@ dependencies {
     implementation("androidx.camera:camera-camera2:1.4.1")
     implementation("androidx.camera:camera-lifecycle:1.4.1")
     implementation("androidx.camera:camera-view:1.4.1")
+    implementation("com.google.mlkit:text-recognition:16.0.1")
     implementation("com.squareup.okhttp3:okhttp:4.12.0")
     implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0")
     implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.7.3")

android-app/app/src/main/java/ru/obdai/receipt/MainActivity.kt

@@ -4,6 +4,7 @@ import android.Manifest
 import android.content.pm.PackageManager
 import android.os.Bundle
 import android.graphics.Bitmap
+import android.graphics.Rect
 import androidx.activity.ComponentActivity
 import androidx.activity.compose.setContent
 import androidx.activity.result.contract.ActivityResultContracts
@@ -19,9 +20,11 @@ import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.fillMaxSize
 import androidx.compose.foundation.layout.fillMaxWidth
 import androidx.compose.foundation.layout.padding
+import androidx.compose.material3.CircularProgressIndicator
 import androidx.compose.material3.Button
 import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
+import androidx.compose.runtime.collectAsState
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.setValue
@@ -34,10 +37,19 @@ import androidx.compose.ui.unit.dp
 import androidx.compose.ui.viewinterop.AndroidView
 import androidx.core.content.ContextCompat
 import ru.obdai.receipt.camera.CameraManager
+import ru.obdai.receipt.crop.CropHelper
+import ru.obdai.receipt.network.ApiClient
+import ru.obdai.receipt.viewmodel.ReceiptViewModel
+import ru.obdai.receipt.viewmodel.UiState
+import androidx.lifecycle.ViewModelProvider
+import androidx.lifecycle.ViewModel
+import androidx.lifecycle.viewmodel.compose.viewModel
 import java.util.concurrent.Executors
 
 class MainActivity : ComponentActivity() {
     private var latestBitmap by mutableStateOf<Bitmap?>(null)
+    private var capturedBitmap by mutableStateOf<Bitmap?>(null)
+    private var detectedBounds by mutableStateOf<Rect?>(null)
     private val cameraExecutor = Executors.newSingleThreadExecutor()
     private val cameraManager = CameraManager()
     private val permissionLauncher = registerForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
@@ -46,7 +58,37 @@ class MainActivity : ComponentActivity() {
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
-        setContent { CameraScreen(latestBitmap, ::onPreviewReady) }
+        setContent {
+            val receiptViewModel: ReceiptViewModel = viewModel(
+                factory = object : ViewModelProvider.Factory {
+                    @Suppress("UNCHECKED_CAST")
+                    override fun <T : ViewModel> create(modelClass: Class<T>): T {
+                        return ReceiptViewModel(
+                            ApiClient(BuildConfig.RECEIPT_API_URL, BuildConfig.RECEIPT_API_TOKEN)
+                        ) as T
+                    }
+                }
+            )
+            CameraScreen(
+                bitmap = latestBitmap,
+                capturedBitmap = capturedBitmap,
+                bounds = detectedBounds,
+                state = receiptViewModel.state.collectAsState().value,
+                onPreviewReady = ::onPreviewReady,
+                onCapture = { capturedBitmap = latestBitmap },
+                onRecognize = { bitmap ->
+                    receiptViewModel.detectZone(bitmap) { bounds ->
+                        detectedBounds = bounds
+                        if (bounds != null) {
+                            receiptViewModel.recognize(
+                                CropHelper.crop(bitmap, bounds),
+                                "Recognize only medicine names, dosage, quantity and schedule. Return concise text."
+                            )
+                        }
+                    }
+                }
+            )
+        }
         if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
             permissionLauncher.launch(Manifest.permission.CAMERA)
         }
@@ -79,13 +121,22 @@ class MainActivity : ComponentActivity() {
 
     override fun onDestroy() {
         latestBitmap?.let { if (!it.isRecycled) it.recycle() }
+        if (capturedBitmap !== latestBitmap) capturedBitmap?.let { if (!it.isRecycled) it.recycle() }
         cameraExecutor.shutdown()
         super.onDestroy()
     }
 }
 
 @Composable
-private fun CameraScreen(bitmap: Bitmap?, onPreviewReady: (PreviewView) -> Unit = {}) {
+private fun CameraScreen(
+    bitmap: Bitmap?,
+    capturedBitmap: Bitmap?,
+    bounds: Rect?,
+    state: UiState,
+    onPreviewReady: (PreviewView) -> Unit,
+    onCapture: () -> Unit,
+    onRecognize: (Bitmap) -> Unit
+) {
     Box(Modifier.fillMaxSize()) {
         AndroidView(
             factory = { context -> PreviewView(context).also(onPreviewReady) },
@@ -95,11 +146,40 @@ private fun CameraScreen(bitmap: Bitmap?, onPreviewReady: (PreviewView) -> Unit
             modifier = Modifier.align(Alignment.BottomCenter).fillMaxWidth().padding(16.dp),
             verticalArrangement = Arrangement.spacedBy(12.dp)
         ) {
-            Button(onClick = { /* Crop and recognize are wired by ReceiptViewModel. */ }, modifier = Modifier.fillMaxWidth()) {
+            Button(onClick = onCapture, modifier = Modifier.fillMaxWidth()) {
-                Text("Распознать")
+                Text("Зафиксировать кадр")
+            }
+            Button(
+                onClick = { capturedBitmap?.let(onRecognize) },
+                enabled = capturedBitmap != null && state !is UiState.Analyzing,
+                modifier = Modifier.fillMaxWidth()
+            ) {
+                if (state is UiState.Analyzing) CircularProgressIndicator()
+                else Text(if (bounds == null) "Найти и распознать препараты" else "Распознать crop")
+            }
+            if (state is UiState.Error) Text(state.message, color = Color.Red)
+            if (state is UiState.Result) Text(state.text, color = Color.Red, modifier = Modifier.padding(24.dp))
+        }
+        if (capturedBitmap != null && bounds != null) {
+            CropOutline(bounds)
+        }
+        if (state is UiState.Result) {
+            ResultOverlay(state.bitmap, state.text)
+        } else if (bitmap != null) {
+            ResultOverlay(bitmap, "")
         }
     }
-        if (bitmap != null) ResultOverlay(bitmap, "")
+}
+
+@Composable
+private fun CropOutline(bounds: Rect) {
+    Canvas(Modifier.fillMaxSize()) {
+        drawRect(
+            color = Color.Green,
+            topLeft = androidx.compose.ui.geometry.Offset(bounds.left.toFloat(), bounds.top.toFloat()),
+            size = androidx.compose.ui.geometry.Size(bounds.width().toFloat(), bounds.height().toFloat()),
+            style = androidx.compose.ui.graphics.drawscope.Stroke(width = 4f)
+        )
     }
 }
 
@@ -108,5 +188,7 @@ private fun ResultOverlay(bitmap: Bitmap, text: String) {
     Canvas(Modifier.fillMaxSize()) {
         drawImage(bitmap.asImageBitmap(), dstSize = IntSize(size.width.toInt(), size.height.toInt()))
     }
+    if (text.isNotBlank()) {
         Text(text, color = Color.Red, modifier = Modifier.padding(24.dp))
     }
+}

android-app/app/src/main/java/ru/obdai/receipt/crop/MedicationZoneDetector.kt

@@ -0,0 +1,31 @@
+package ru.obdai.receipt.crop
+
+import android.graphics.Bitmap
+import android.graphics.Rect
+import com.google.mlkit.vision.common.InputImage
+import com.google.mlkit.vision.text.TextRecognition
+import com.google.mlkit.vision.text.latin.TextRecognizerOptions
+import kotlinx.coroutines.suspendCancellableCoroutine
+import kotlin.coroutines.resume
+
+class MedicationZoneDetector {
+    private val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
+
+    suspend fun detect(bitmap: Bitmap): Rect? = suspendCancellableCoroutine { continuation ->
+        recognizer.process(InputImage.fromBitmap(bitmap, 0))
+            .addOnSuccessListener { result ->
+                val blocks = result.textBlocks
+                    .map { it.boundingBox }
+                    .filterNotNull()
+                    .filter { it.top > bitmap.height / 5 }
+                continuation.resume(blocks.reduceOrNull { first, next ->
+                    Rect(first).apply { union(next) }
+                })
+            }
+            .addOnFailureListener { continuation.resume(null) }
+    }
+
+    fun close() {
+        recognizer.close()
+    }
+}

android-app/app/src/main/java/ru/obdai/receipt/viewmodel/ReceiptViewModel.kt

@@ -7,6 +7,7 @@ import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.flow.MutableStateFlow
 import kotlinx.coroutines.flow.StateFlow
 import kotlinx.coroutines.launch
+import ru.obdai.receipt.crop.MedicationZoneDetector
 import ru.obdai.receipt.network.ApiClient
 
 sealed interface UiState {
@@ -16,7 +17,10 @@ sealed interface UiState {
     data class Error(val message: String) : UiState
 }
 
-class ReceiptViewModel(private val apiClient: ApiClient) : ViewModel() {
+class ReceiptViewModel(
+    private val apiClient: ApiClient,
+    private val zoneDetector: MedicationZoneDetector = MedicationZoneDetector()
+) : ViewModel() {
     private val _state = MutableStateFlow<UiState>(UiState.Idle)
     val state: StateFlow<UiState> = _state
 
@@ -35,17 +39,25 @@ class ReceiptViewModel(private val apiClient: ApiClient) : ViewModel() {
         }
     }
 
+    fun detectZone(bitmap: Bitmap, onDetected: (android.graphics.Rect?) -> Unit) {
+        viewModelScope.launch(Dispatchers.Default) {
+            onDetected(zoneDetector.detect(bitmap))
+        }
+    }
+
     override fun onCleared() {
         (_state.value as? UiState.Result)?.bitmap?.let { bitmap ->
             if (!bitmap.isRecycled) bitmap.recycle()
         }
         _state.value = UiState.Idle
+        zoneDetector.close()
         super.onCleared()
     }
 
     private fun Bitmap.toJpegBytes(): ByteArray {
-        val output = java.io.ByteArrayOutputStream()
+        return java.io.ByteArrayOutputStream().use { output ->
             compress(Bitmap.CompressFormat.JPEG, 92, output)
-        return output.toByteArray()
+            output.toByteArray()
+        }
     }
 }