formed/recipe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement local medication crop recognition flow

Browse Source
This commit is contained in:
“Naeel”
2026-08-29 15:52:54 +03:00
parent 3bcff192e6
commit 3909650177
4 changed files with 138 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
android-app/app/build.gradle.kts
+3 -2
View File
@@ -12,8 +12,8 @@ android {
applicationId = "ru.obdai.receipt"
minSdk = 26
targetSdk = 35
versionCode = 2
versionName = "0.1.1"
versionCode = 4
versionName = "0.1.3"
buildConfigField("String", "RECEIPT_API_TOKEN", "\"\"")
buildConfigField("String", "RECEIPT_API_URL", "\"https://obdai.ru/receipt\"")
@@ -47,6 +47,7 @@ dependencies {
implementation("androidx.camera:camera-camera2:1.4.1")
implementation("androidx.camera:camera-lifecycle:1.4.1")
implementation("androidx.camera:camera-view:1.4.1")
implementation("com.google.mlkit:text-recognition:16.0.1")
implementation("com.squareup.okhttp3:okhttp:4.12.0")
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0")
implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.7.3")
android-app/app/src/main/java/ru/obdai/receipt/MainActivity.kt
+88 -6
View File
@@ -4,6 +4,7 @@ import android.Manifest
import android.content.pm.PackageManager
import android.os.Bundle
import android.graphics.Bitmap
import android.graphics.Rect
import androidx.activity.ComponentActivity
import androidx.activity.compose.setContent
import androidx.activity.result.contract.ActivityResultContracts
@@ -19,9 +20,11 @@ import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Button
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue
@@ -34,10 +37,19 @@ import androidx.compose.ui.unit.dp
import androidx.compose.ui.viewinterop.AndroidView
import androidx.core.content.ContextCompat
import ru.obdai.receipt.camera.CameraManager
import ru.obdai.receipt.crop.CropHelper
import ru.obdai.receipt.network.ApiClient
import ru.obdai.receipt.viewmodel.ReceiptViewModel
import ru.obdai.receipt.viewmodel.UiState
import androidx.lifecycle.ViewModelProvider
import androidx.lifecycle.ViewModel
import androidx.lifecycle.viewmodel.compose.viewModel
import java.util.concurrent.Executors
class MainActivity : ComponentActivity() {
private var latestBitmap by mutableStateOf<Bitmap?>(null)
private var capturedBitmap by mutableStateOf<Bitmap?>(null)
private var detectedBounds by mutableStateOf<Rect?>(null)
private val cameraExecutor = Executors.newSingleThreadExecutor()
private val cameraManager = CameraManager()
private val permissionLauncher = registerForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
@@ -46,7 +58,37 @@ class MainActivity : ComponentActivity() {
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
setContent { CameraScreen(latestBitmap, ::onPreviewReady) }
setContent {
val receiptViewModel: ReceiptViewModel = viewModel(
factory = object : ViewModelProvider.Factory {
@Suppress("UNCHECKED_CAST")
override fun <T : ViewModel> create(modelClass: Class<T>): T {
return ReceiptViewModel(
ApiClient(BuildConfig.RECEIPT_API_URL, BuildConfig.RECEIPT_API_TOKEN)
) as T
}
}
)
CameraScreen(
bitmap = latestBitmap,
capturedBitmap = capturedBitmap,
bounds = detectedBounds,
state = receiptViewModel.state.collectAsState().value,
onPreviewReady = ::onPreviewReady,
onCapture = { capturedBitmap = latestBitmap },
onRecognize = { bitmap ->
receiptViewModel.detectZone(bitmap) { bounds ->
detectedBounds = bounds
if (bounds != null) {
receiptViewModel.recognize(
CropHelper.crop(bitmap, bounds),
"Recognize only medicine names, dosage, quantity and schedule. Return concise text."
)
}
}
}
)
}
if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
permissionLauncher.launch(Manifest.permission.CAMERA)
}
@@ -79,13 +121,22 @@ class MainActivity : ComponentActivity() {
override fun onDestroy() {
latestBitmap?.let { if (!it.isRecycled) it.recycle() }
if (capturedBitmap !== latestBitmap) capturedBitmap?.let { if (!it.isRecycled) it.recycle() }
cameraExecutor.shutdown()
super.onDestroy()
}
}
@Composable
private fun CameraScreen(bitmap: Bitmap?, onPreviewReady: (PreviewView) -> Unit = {}) {
private fun CameraScreen(
bitmap: Bitmap?,
capturedBitmap: Bitmap?,
bounds: Rect?,
state: UiState,
onPreviewReady: (PreviewView) -> Unit,
onCapture: () -> Unit,
onRecognize: (Bitmap) -> Unit
) {
Box(Modifier.fillMaxSize()) {
AndroidView(
factory = { context -> PreviewView(context).also(onPreviewReady) },
@@ -95,11 +146,40 @@ private fun CameraScreen(bitmap: Bitmap?, onPreviewReady: (PreviewView) -> Unit
modifier = Modifier.align(Alignment.BottomCenter).fillMaxWidth().padding(16.dp),
verticalArrangement = Arrangement.spacedBy(12.dp)
) {
Button(onClick = { /* Crop and recognize are wired by ReceiptViewModel. */ }, modifier = Modifier.fillMaxWidth()) {
Text("Распознать")
Button(onClick = onCapture, modifier = Modifier.fillMaxWidth()) {
Text("Зафиксировать кадр")
}
Button(
onClick = { capturedBitmap?.let(onRecognize) },
enabled = capturedBitmap != null && state !is UiState.Analyzing,
modifier = Modifier.fillMaxWidth()
) {
if (state is UiState.Analyzing) CircularProgressIndicator()
else Text(if (bounds == null) "Найти и распознать препараты" else "Распознать crop")
}
if (state is UiState.Error) Text(state.message, color = Color.Red)
if (state is UiState.Result) Text(state.text, color = Color.Red, modifier = Modifier.padding(24.dp))
}
if (bitmap != null) ResultOverlay(bitmap, "")
if (capturedBitmap != null && bounds != null) {
CropOutline(bounds)
}
if (state is UiState.Result) {
ResultOverlay(state.bitmap, state.text)
} else if (bitmap != null) {
ResultOverlay(bitmap, "")
}
}
}
@Composable
private fun CropOutline(bounds: Rect) {
Canvas(Modifier.fillMaxSize()) {
drawRect(
color = Color.Green,
topLeft = androidx.compose.ui.geometry.Offset(bounds.left.toFloat(), bounds.top.toFloat()),
size = androidx.compose.ui.geometry.Size(bounds.width().toFloat(), bounds.height().toFloat()),
style = androidx.compose.ui.graphics.drawscope.Stroke(width = 4f)
)
}
}
@@ -108,5 +188,7 @@ private fun ResultOverlay(bitmap: Bitmap, text: String) {
Canvas(Modifier.fillMaxSize()) {
drawImage(bitmap.asImageBitmap(), dstSize = IntSize(size.width.toInt(), size.height.toInt()))
}
Text(text, color = Color.Red, modifier = Modifier.padding(24.dp))
if (text.isNotBlank()) {
Text(text, color = Color.Red, modifier = Modifier.padding(24.dp))
}
}
android-app/app/src/main/java/ru/obdai/receipt/crop/MedicationZoneDetector.kt
+31
View File
@@ -0,0 +1,31 @@
package ru.obdai.receipt.crop
import android.graphics.Bitmap
import android.graphics.Rect
import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.text.TextRecognition
import com.google.mlkit.vision.text.latin.TextRecognizerOptions
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlin.coroutines.resume
class MedicationZoneDetector {
private val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
suspend fun detect(bitmap: Bitmap): Rect? = suspendCancellableCoroutine { continuation ->
recognizer.process(InputImage.fromBitmap(bitmap, 0))
.addOnSuccessListener { result ->
val blocks = result.textBlocks
.map { it.boundingBox }
.filterNotNull()
.filter { it.top > bitmap.height / 5 }
continuation.resume(blocks.reduceOrNull { first, next ->
Rect(first).apply { union(next) }
})
}
.addOnFailureListener { continuation.resume(null) }
}
fun close() {
recognizer.close()
}
}
android-app/app/src/main/java/ru/obdai/receipt/viewmodel/ReceiptViewModel.kt
+16 -4
View File
@@ -7,6 +7,7 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.launch
import ru.obdai.receipt.crop.MedicationZoneDetector
import ru.obdai.receipt.network.ApiClient
sealed interface UiState {
@@ -16,7 +17,10 @@ sealed interface UiState {
data class Error(val message: String) : UiState
}
class ReceiptViewModel(private val apiClient: ApiClient) : ViewModel() {
class ReceiptViewModel(
private val apiClient: ApiClient,
private val zoneDetector: MedicationZoneDetector = MedicationZoneDetector()
) : ViewModel() {
private val _state = MutableStateFlow<UiState>(UiState.Idle)
val state: StateFlow<UiState> = _state
@@ -35,17 +39,25 @@ class ReceiptViewModel(private val apiClient: ApiClient) : ViewModel() {
}
}
fun detectZone(bitmap: Bitmap, onDetected: (android.graphics.Rect?) -> Unit) {
viewModelScope.launch(Dispatchers.Default) {
onDetected(zoneDetector.detect(bitmap))
}
}
override fun onCleared() {
(_state.value as? UiState.Result)?.bitmap?.let { bitmap ->
if (!bitmap.isRecycled) bitmap.recycle()
}
_state.value = UiState.Idle
zoneDetector.close()
super.onCleared()
}
private fun Bitmap.toJpegBytes(): ByteArray {
val output = java.io.ByteArrayOutputStream()
compress(Bitmap.CompressFormat.JPEG, 92, output)
return output.toByteArray()
return java.io.ByteArrayOutputStream().use { output ->
compress(Bitmap.CompressFormat.JPEG, 92, output)
output.toByteArray()
}
}
}