[WIP] feat: Android Impeller support #1283
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,122 @@ | ||
| package dev.steenbakker.mobile_scanner | ||
|
|
||
| import android.app.Activity | ||
| import android.content.BroadcastReceiver | ||
| import android.content.Context | ||
| import android.content.Intent | ||
| import android.content.IntentFilter | ||
| import android.content.res.Configuration | ||
| import android.os.Build | ||
| import android.os.Handler | ||
| import android.os.Looper | ||
| import android.view.Display | ||
| import android.view.Surface | ||
| import android.view.WindowManager | ||
| import dev.steenbakker.mobile_scanner.utils.serialize | ||
| import io.flutter.embedding.engine.systemchannels.PlatformChannel | ||
| import io.flutter.plugin.common.EventChannel | ||
|
|
||
| /** | ||
| * This class will listen to device orientation changes. | ||
| * | ||
| * When a new orientation is received, the registered listener will be invoked. | ||
| */ | ||
| class DeviceOrientationListener( | ||
| private val activity: Activity, | ||
| ): BroadcastReceiver(), EventChannel.StreamHandler { | ||
|
|
||
| companion object { | ||
| // The intent filter for listening to orientation changes. | ||
| private val orientationIntentFilter = IntentFilter(Intent.ACTION_CONFIGURATION_CHANGED) | ||
| } | ||
|
|
||
| // The event sink that handles device orientation events. | ||
| private var deviceOrientationEventSink: EventChannel.EventSink? = null | ||
|
|
||
| // The last received orientation. This is used to prevent duplicate events. | ||
| private var lastOrientation: PlatformChannel.DeviceOrientation? = null | ||
| // Whether the device orientation is currently being observed. | ||
| private var listening = false | ||
|
|
||
| override fun onReceive(context: Context?, intent: Intent?) { | ||
| val orientation: PlatformChannel.DeviceOrientation = getUIOrientation() | ||
| if (orientation != lastOrientation) { | ||
| Handler(Looper.getMainLooper()).post { | ||
| deviceOrientationEventSink?.success(orientation.serialize()) | ||
| } | ||
| } | ||
|
|
||
| lastOrientation = orientation | ||
| } | ||
|
|
||
| override fun onListen(event: Any?, eventSink: EventChannel.EventSink?) { | ||
| deviceOrientationEventSink = eventSink | ||
| } | ||
|
|
||
| override fun onCancel(event: Any?) { | ||
| deviceOrientationEventSink = null | ||
| } | ||
|
|
||
| @Suppress("deprecation") | ||
| private fun getDisplay(): Display { | ||
| return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { | ||
| activity.display!! | ||
| } else { | ||
| (activity.getSystemService(Context.WINDOW_SERVICE) as WindowManager).defaultDisplay | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Gets the current user interface orientation. | ||
| */ | ||
| fun getUIOrientation(): PlatformChannel.DeviceOrientation { | ||
| val rotation: Int = getDisplay().rotation | ||
| val orientation: Int = activity.resources.configuration.orientation | ||
|
|
||
| return when(orientation) { | ||
| Configuration.ORIENTATION_PORTRAIT -> { | ||
| if (rotation == Surface.ROTATION_0 || rotation == Surface.ROTATION_90) { | ||
| PlatformChannel.DeviceOrientation.PORTRAIT_UP | ||
| } else { | ||
| PlatformChannel.DeviceOrientation.PORTRAIT_DOWN | ||
| } | ||
| } | ||
| Configuration.ORIENTATION_LANDSCAPE -> { | ||
| if (rotation == Surface.ROTATION_0 || rotation == Surface.ROTATION_90) { | ||
| PlatformChannel.DeviceOrientation.LANDSCAPE_LEFT | ||
| } else { | ||
| PlatformChannel.DeviceOrientation.LANDSCAPE_RIGHT | ||
| } | ||
| } | ||
| Configuration.ORIENTATION_UNDEFINED -> PlatformChannel.DeviceOrientation.PORTRAIT_UP | ||
| else -> PlatformChannel.DeviceOrientation.PORTRAIT_UP | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Start listening to device orientation changes. | ||
| */ | ||
| fun start() { | ||
| if (listening) { | ||
| return | ||
| } | ||
|
|
||
| listening = true | ||
| activity.registerReceiver(this, orientationIntentFilter) | ||
|
|
||
| // Trigger the orientation listener with the current value. | ||
| onReceive(activity, null) | ||
| } | ||
|
|
||
| /** | ||
| * Stop listening to device orientation changes. | ||
| */ | ||
| fun stop() { | ||
| if (!listening) { | ||
| return | ||
| } | ||
|
|
||
| activity.unregisterReceiver(this) | ||
| listening = false | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -22,6 +22,7 @@ import androidx.camera.core.ExperimentalGetImage | |
| import androidx.camera.core.ImageAnalysis | ||
| import androidx.camera.core.ImageProxy | ||
| import androidx.camera.core.Preview | ||
| import androidx.camera.core.SurfaceRequest | ||
| import androidx.camera.core.TorchState | ||
| import androidx.camera.core.resolutionselector.ResolutionSelector | ||
| import androidx.camera.core.resolutionselector.ResolutionStrategy | ||
|
|
@@ -37,20 +38,23 @@ import dev.steenbakker.mobile_scanner.objects.DetectionSpeed | |
| import dev.steenbakker.mobile_scanner.objects.MobileScannerErrorCodes | ||
| import dev.steenbakker.mobile_scanner.objects.MobileScannerStartParameters | ||
| import dev.steenbakker.mobile_scanner.utils.YuvToRgbConverter | ||
| import dev.steenbakker.mobile_scanner.utils.serialize | ||
| import io.flutter.view.TextureRegistry | ||
| import kotlinx.coroutines.CoroutineScope | ||
| import kotlinx.coroutines.Dispatchers | ||
| import kotlinx.coroutines.coroutineScope | ||
| import kotlinx.coroutines.launch | ||
| import java.io.ByteArrayOutputStream | ||
| import java.io.IOException | ||
| import java.util.concurrent.Executors | ||
| import kotlin.math.roundToInt | ||
|
|
||
| class MobileScanner( | ||
| private val activity: Activity, | ||
| private val textureRegistry: TextureRegistry, | ||
| private val mobileScannerCallback: MobileScannerCallback, | ||
| private val mobileScannerErrorCallback: MobileScannerErrorCallback, | ||
| private val deviceOrientationListener: DeviceOrientationListener, | ||
| private val barcodeScannerFactory: (options: BarcodeScannerOptions?) -> BarcodeScanner = ::defaultBarcodeScannerFactory, | ||
| ) { | ||
|
|
||
|
|
@@ -59,7 +63,7 @@ class MobileScanner( | |
| private var camera: Camera? = null | ||
| private var cameraSelector: CameraSelector? = null | ||
| private var preview: Preview? = null | ||
| private var surfaceProducer: TextureRegistry.SurfaceProducer? = null | ||
| private var scanner: BarcodeScanner? = null | ||
| private var lastScanned: List<String?>? = null | ||
| private var scannerTimeout = false | ||
|
|
@@ -189,6 +193,70 @@ class MobileScanner( | |
| } | ||
| } | ||
|
|
||
| /** | ||
| * Create a {@link Preview.SurfaceProvider} that specifies how to provide a {@link Surface} to a | ||
| * {@code Preview}. | ||
| */ | ||
| @VisibleForTesting | ||
| fun createSurfaceProvider(surfaceProducer: TextureRegistry.SurfaceProducer): Preview.SurfaceProvider { | ||
| return Preview.SurfaceProvider { | ||
| request: SurfaceRequest -> | ||
| run { | ||
| // Set the callback for the surfaceProducer to invalidate Surfaces that it produces | ||
| // when they get destroyed. | ||
| surfaceProducer.setCallback( | ||
| object : TextureRegistry.SurfaceProducer.Callback { | ||
| override fun onSurfaceAvailable() { | ||
| // Do nothing. The Preview.SurfaceProvider will handle this | ||
| // whenever a new Surface is needed. | ||
| } | ||
|
|
||
| // TODO: replace with "onSurfaceCleanup" when available in Flutter 3.28 or later | ||
| // See https://github.com/flutter/flutter/pull/160937 | ||
| override fun onSurfaceDestroyed() { | ||
| // Invalidate the SurfaceRequest so that CameraX knows to to make a new request | ||
| // for a surface. | ||
| request.invalidate() | ||
| } | ||
| } | ||
| ) | ||
|
|
||
| // Provide the surface. | ||
| surfaceProducer.setSize(request.resolution.width, request.resolution.height) | ||
|
|
||
| val surface: Surface = surfaceProducer.surface | ||
|
|
||
| // The single thread executor is only used to invoke the result callback. | ||
| // Thus it is safe to use a new executor, | ||
| // instead of reusing the executor that is passed to the camera process provider. | ||
| request.provideSurface(surface, Executors.newSingleThreadExecutor()) { | ||
| // Handle the result of the request for a surface. | ||
| // See: https://developer.android.com/reference/androidx/camera/core/SurfaceRequest.Result | ||
|
|
||
| // Always attempt a release. | ||
| surface.release() | ||
|
|
||
| val resultCode: Int = it.resultCode | ||
|
|
||
| when(resultCode) { | ||
| SurfaceRequest.Result.RESULT_REQUEST_CANCELLED, | ||
| SurfaceRequest.Result.RESULT_WILL_NOT_PROVIDE_SURFACE, | ||
| SurfaceRequest.Result.RESULT_SURFACE_ALREADY_PROVIDED, | ||
| SurfaceRequest.Result.RESULT_SURFACE_USED_SUCCESSFULLY -> { | ||
| // Only need to release, do nothing. | ||
| } | ||
| SurfaceRequest.Result.RESULT_INVALID_SURFACE -> { | ||
| // The surface was invalid, so it is not clear how to recover from this. | ||
| } | ||
| else -> { | ||
| // Fallthrough, in case any result codes are added later. | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private fun rotateBitmap(bitmap: Bitmap, degrees: Float): Bitmap { | ||
| val matrix = Matrix() | ||
| matrix.postRotate(degrees) | ||
|
|
@@ -249,19 +317,22 @@ class MobileScanner( | |
| this.returnImage = returnImage | ||
| this.invertImage = invertImage | ||
|
|
||
| if (camera?.cameraInfo != null && preview != null && surfaceProducer != null && !isPaused) { | ||
|
|
||
| // TODO: resume here for seamless transition | ||
| // if (isPaused) { | ||
| // resumeCamera() | ||
| // mobileScannerStartedCallback( | ||
| // MobileScannerStartParameters( | ||
| // if (portrait) width else height, | ||
| // if (portrait) height else width, | ||
| // deviceOrientationListener.getUIOrientation().serialize(), | ||
| // sensorRotationDegrees, | ||
| // surfaceProducer!!.handlesCropAndRotation(), | ||
| // currentTorchState, | ||
| // surfaceProducer!!.id(), | ||
| // numberOfCameras ?: 0 | ||
| // ) | ||
| // ) | ||
| // return | ||
| // } | ||
|
|
@@ -287,23 +358,10 @@ class MobileScanner( | |
| } | ||
|
|
||
| cameraProvider?.unbindAll() | ||
| surfaceProducer = surfaceProducer ?: textureRegistry.createSurfaceProducer() | ||
|
There was a problem hiding this comment. @juliansteenbakker The There was a problem hiding this comment. The ?: was added because otherwise a new texture was created instead of the existing one being used after resuming from pause. |
||
| val surfaceProvider: Preview.SurfaceProvider = createSurfaceProvider(surfaceProducer!!) | ||
|
|
||
| // Preview | ||
|
There was a problem hiding this comment. We no longer have this check, I'm not sure if we should put it back or not. |
||
|
There was a problem hiding this comment. This is replaced by the |
||
|
|
||
| // Build the preview to be shown on the Flutter texture | ||
| val previewBuilder = Preview.Builder() | ||
|
|
@@ -384,7 +442,8 @@ class MobileScanner( | |
| val resolution = analysis.resolutionInfo!!.resolution | ||
| val width = resolution.width.toDouble() | ||
| val height = resolution.height.toDouble() | ||
| val sensorRotationDegrees = camera?.cameraInfo?.sensorRotationDegrees ?: 0 | ||
| val portrait = sensorRotationDegrees % 180 == 0 | ||
|
|
||
| // Start with 'unavailable' torch state. | ||
| var currentTorchState: Int = -1 | ||
|
|
@@ -397,12 +456,17 @@ class MobileScanner( | |
| currentTorchState = it.torchState.value ?: -1 | ||
| } | ||
|
|
||
| deviceOrientationListener.start() | ||
|
|
||
| mobileScannerStartedCallback( | ||
| MobileScannerStartParameters( | ||
| if (portrait) width else height, | ||
| if (portrait) height else width, | ||
| deviceOrientationListener.getUIOrientation().serialize(), | ||
| sensorRotationDegrees, | ||
| surfaceProducer!!.handlesCropAndRotation(), | ||
| currentTorchState, | ||
| surfaceProducer!!.id(), | ||
| numberOfCameras ?: 0 | ||
| ) | ||
| ) | ||
|
|
@@ -420,6 +484,7 @@ class MobileScanner( | |
| throw AlreadyStopped() | ||
| } | ||
|
|
||
| deviceOrientationListener.stop() | ||
| pauseCamera() | ||
| } | ||
|
|
||
|
|
@@ -431,6 +496,7 @@ class MobileScanner( | |
| throw AlreadyStopped() | ||
| } | ||
|
|
||
| deviceOrientationListener.stop() | ||
| releaseCamera() | ||
| } | ||
|
|
||
|
|
@@ -469,8 +535,9 @@ class MobileScanner( | |
| // The camera will be closed when the last use case is unbound. | ||
| cameraProvider?.unbindAll() | ||
|
|
||
| // Release the surface for the preview. | ||
| surfaceProducer?.release() | ||
| surfaceProducer = null | ||
|
|
||
| // Release the scanner. | ||
| scanner?.close() | ||
|
|
||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think that adding this line fixes any occurrences of
A resource failed to call closethat we sometimes get.Previously, we never did anything with the result callback, thus also not releasing the surface.