diff --git a/android-app/app/build.gradle.kts b/android-app/app/build.gradle.kts index 13ac973..edff92a 100644 --- a/android-app/app/build.gradle.kts +++ b/android-app/app/build.gradle.kts @@ -12,8 +12,8 @@ android { applicationId = "ru.obdai.receipt" minSdk = 26 targetSdk = 35 - versionCode = 2 - versionName = "0.1.1" + versionCode = 4 + versionName = "0.1.3" buildConfigField("String", "RECEIPT_API_TOKEN", "\"\"") buildConfigField("String", "RECEIPT_API_URL", "\"https://obdai.ru/receipt\"") @@ -47,6 +47,7 @@ dependencies { implementation("androidx.camera:camera-camera2:1.4.1") implementation("androidx.camera:camera-lifecycle:1.4.1") implementation("androidx.camera:camera-view:1.4.1") + implementation("com.google.mlkit:text-recognition:16.0.1") implementation("com.squareup.okhttp3:okhttp:4.12.0") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0") implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.7.3") diff --git a/android-app/app/src/main/java/ru/obdai/receipt/MainActivity.kt b/android-app/app/src/main/java/ru/obdai/receipt/MainActivity.kt index 15a1dd3..f0859b2 100644 --- a/android-app/app/src/main/java/ru/obdai/receipt/MainActivity.kt +++ b/android-app/app/src/main/java/ru/obdai/receipt/MainActivity.kt @@ -4,6 +4,7 @@ import android.Manifest import android.content.pm.PackageManager import android.os.Bundle import android.graphics.Bitmap +import android.graphics.Rect import androidx.activity.ComponentActivity import androidx.activity.compose.setContent import androidx.activity.result.contract.ActivityResultContracts @@ -19,9 +20,11 @@ import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding +import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.Button import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.setValue @@ -34,10 +37,19 @@ import androidx.compose.ui.unit.dp import androidx.compose.ui.viewinterop.AndroidView import androidx.core.content.ContextCompat import ru.obdai.receipt.camera.CameraManager +import ru.obdai.receipt.crop.CropHelper +import ru.obdai.receipt.network.ApiClient +import ru.obdai.receipt.viewmodel.ReceiptViewModel +import ru.obdai.receipt.viewmodel.UiState +import androidx.lifecycle.ViewModelProvider +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewmodel.compose.viewModel import java.util.concurrent.Executors class MainActivity : ComponentActivity() { private var latestBitmap by mutableStateOf(null) + private var capturedBitmap by mutableStateOf(null) + private var detectedBounds by mutableStateOf(null) private val cameraExecutor = Executors.newSingleThreadExecutor() private val cameraManager = CameraManager() private val permissionLauncher = registerForActivityResult(ActivityResultContracts.RequestPermission()) { granted -> @@ -46,7 +58,37 @@ class MainActivity : ComponentActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - setContent { CameraScreen(latestBitmap, ::onPreviewReady) } + setContent { + val receiptViewModel: ReceiptViewModel = viewModel( + factory = object : ViewModelProvider.Factory { + @Suppress("UNCHECKED_CAST") + override fun create(modelClass: Class): T { + return ReceiptViewModel( + ApiClient(BuildConfig.RECEIPT_API_URL, BuildConfig.RECEIPT_API_TOKEN) + ) as T + } + } + ) + CameraScreen( + bitmap = latestBitmap, + capturedBitmap = capturedBitmap, + bounds = detectedBounds, + state = receiptViewModel.state.collectAsState().value, + onPreviewReady = ::onPreviewReady, + onCapture = { capturedBitmap = latestBitmap }, + onRecognize = { bitmap -> + receiptViewModel.detectZone(bitmap) { bounds -> + detectedBounds = bounds + if (bounds != null) { + receiptViewModel.recognize( + CropHelper.crop(bitmap, bounds), + "Recognize only medicine names, dosage, quantity and schedule. Return concise text." + ) + } + } + } + ) + } if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) { permissionLauncher.launch(Manifest.permission.CAMERA) } @@ -79,13 +121,22 @@ class MainActivity : ComponentActivity() { override fun onDestroy() { latestBitmap?.let { if (!it.isRecycled) it.recycle() } + if (capturedBitmap !== latestBitmap) capturedBitmap?.let { if (!it.isRecycled) it.recycle() } cameraExecutor.shutdown() super.onDestroy() } } @Composable -private fun CameraScreen(bitmap: Bitmap?, onPreviewReady: (PreviewView) -> Unit = {}) { +private fun CameraScreen( + bitmap: Bitmap?, + capturedBitmap: Bitmap?, + bounds: Rect?, + state: UiState, + onPreviewReady: (PreviewView) -> Unit, + onCapture: () -> Unit, + onRecognize: (Bitmap) -> Unit +) { Box(Modifier.fillMaxSize()) { AndroidView( factory = { context -> PreviewView(context).also(onPreviewReady) }, @@ -95,11 +146,40 @@ private fun CameraScreen(bitmap: Bitmap?, onPreviewReady: (PreviewView) -> Unit modifier = Modifier.align(Alignment.BottomCenter).fillMaxWidth().padding(16.dp), verticalArrangement = Arrangement.spacedBy(12.dp) ) { - Button(onClick = { /* Crop and recognize are wired by ReceiptViewModel. */ }, modifier = Modifier.fillMaxWidth()) { - Text("Распознать") + Button(onClick = onCapture, modifier = Modifier.fillMaxWidth()) { + Text("Зафиксировать кадр") } + Button( + onClick = { capturedBitmap?.let(onRecognize) }, + enabled = capturedBitmap != null && state !is UiState.Analyzing, + modifier = Modifier.fillMaxWidth() + ) { + if (state is UiState.Analyzing) CircularProgressIndicator() + else Text(if (bounds == null) "Найти и распознать препараты" else "Распознать crop") + } + if (state is UiState.Error) Text(state.message, color = Color.Red) + if (state is UiState.Result) Text(state.text, color = Color.Red, modifier = Modifier.padding(24.dp)) } - if (bitmap != null) ResultOverlay(bitmap, "") + if (capturedBitmap != null && bounds != null) { + CropOutline(bounds) + } + if (state is UiState.Result) { + ResultOverlay(state.bitmap, state.text) + } else if (bitmap != null) { + ResultOverlay(bitmap, "") + } + } +} + +@Composable +private fun CropOutline(bounds: Rect) { + Canvas(Modifier.fillMaxSize()) { + drawRect( + color = Color.Green, + topLeft = androidx.compose.ui.geometry.Offset(bounds.left.toFloat(), bounds.top.toFloat()), + size = androidx.compose.ui.geometry.Size(bounds.width().toFloat(), bounds.height().toFloat()), + style = androidx.compose.ui.graphics.drawscope.Stroke(width = 4f) + ) } } @@ -108,5 +188,7 @@ private fun ResultOverlay(bitmap: Bitmap, text: String) { Canvas(Modifier.fillMaxSize()) { drawImage(bitmap.asImageBitmap(), dstSize = IntSize(size.width.toInt(), size.height.toInt())) } - Text(text, color = Color.Red, modifier = Modifier.padding(24.dp)) + if (text.isNotBlank()) { + Text(text, color = Color.Red, modifier = Modifier.padding(24.dp)) + } } diff --git a/android-app/app/src/main/java/ru/obdai/receipt/crop/MedicationZoneDetector.kt b/android-app/app/src/main/java/ru/obdai/receipt/crop/MedicationZoneDetector.kt new file mode 100644 index 0000000..c0849fc --- /dev/null +++ b/android-app/app/src/main/java/ru/obdai/receipt/crop/MedicationZoneDetector.kt @@ -0,0 +1,31 @@ +package ru.obdai.receipt.crop + +import android.graphics.Bitmap +import android.graphics.Rect +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.text.TextRecognition +import com.google.mlkit.vision.text.latin.TextRecognizerOptions +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlin.coroutines.resume + +class MedicationZoneDetector { + private val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS) + + suspend fun detect(bitmap: Bitmap): Rect? = suspendCancellableCoroutine { continuation -> + recognizer.process(InputImage.fromBitmap(bitmap, 0)) + .addOnSuccessListener { result -> + val blocks = result.textBlocks + .map { it.boundingBox } + .filterNotNull() + .filter { it.top > bitmap.height / 5 } + continuation.resume(blocks.reduceOrNull { first, next -> + Rect(first).apply { union(next) } + }) + } + .addOnFailureListener { continuation.resume(null) } + } + + fun close() { + recognizer.close() + } +} \ No newline at end of file diff --git a/android-app/app/src/main/java/ru/obdai/receipt/viewmodel/ReceiptViewModel.kt b/android-app/app/src/main/java/ru/obdai/receipt/viewmodel/ReceiptViewModel.kt index a164ff2..211f032 100644 --- a/android-app/app/src/main/java/ru/obdai/receipt/viewmodel/ReceiptViewModel.kt +++ b/android-app/app/src/main/java/ru/obdai/receipt/viewmodel/ReceiptViewModel.kt @@ -7,6 +7,7 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.launch +import ru.obdai.receipt.crop.MedicationZoneDetector import ru.obdai.receipt.network.ApiClient sealed interface UiState { @@ -16,7 +17,10 @@ sealed interface UiState { data class Error(val message: String) : UiState } -class ReceiptViewModel(private val apiClient: ApiClient) : ViewModel() { +class ReceiptViewModel( + private val apiClient: ApiClient, + private val zoneDetector: MedicationZoneDetector = MedicationZoneDetector() +) : ViewModel() { private val _state = MutableStateFlow(UiState.Idle) val state: StateFlow = _state @@ -35,17 +39,25 @@ class ReceiptViewModel(private val apiClient: ApiClient) : ViewModel() { } } + fun detectZone(bitmap: Bitmap, onDetected: (android.graphics.Rect?) -> Unit) { + viewModelScope.launch(Dispatchers.Default) { + onDetected(zoneDetector.detect(bitmap)) + } + } + override fun onCleared() { (_state.value as? UiState.Result)?.bitmap?.let { bitmap -> if (!bitmap.isRecycled) bitmap.recycle() } _state.value = UiState.Idle + zoneDetector.close() super.onCleared() } private fun Bitmap.toJpegBytes(): ByteArray { - val output = java.io.ByteArrayOutputStream() - compress(Bitmap.CompressFormat.JPEG, 92, output) - return output.toByteArray() + return java.io.ByteArrayOutputStream().use { output -> + compress(Bitmap.CompressFormat.JPEG, 92, output) + output.toByteArray() + } } }