You can control the frame rate of a virtual display by
- Creating an ImageReader.
- Getting the surface from an ImageReader and submitting it to the VirtualDisplay.
- On the ImageReader callback, acquire the latest Image frame, and store it.
- In another thread/event tick grab that runs at your desired frame rate, grab the latest frame from the intermedia variable.
- Take the surface of the encoder and grab the hardware canvas.
- Convert the Image to a Bitmap.
- Draw the Bitmap to the hardware canvas (optionally downscale or other drawing things you want), then unlock it.
You'll need to close images that aren't actually drawn, but this will be obvious as the application will crash if you don't do this but the stack trace will allow you to find the solution (along with the documentation) pretty fast.
This will work with recent versions of Android and is measured at around 11ms per frame. Most of the time spent will be in conversion of Image -> Bitmap which requires the pixels being downloaded from the GPU -> CPU, but then drawing will be done via GPU and very is fast.
If you are working with Android 31 and above you can replace steps 5-7 with an ImageWriter which will keep everything in the GPU. However prior to API 31 the ImageWriter will not be able to write to an encoder surface it expects pixel format RGB888 and the VirtualDisplay will produce images in pixel format RGBA8888 space and the work around for this was introduced in API 31.
Keep in mind that using pure EGL Surface's may be faster. But this works well enough to work at 30 fps. Multiple threads may be able to get it working at 60fps.
Here is some example Kotlin code that does this:
import android.annotation.SuppressLint
import android.graphics.Bitmap
import android.graphics.Paint
import android.graphics.PixelFormat
import android.graphics.Rect
import android.media.Image
import android.media.ImageReader
import android.view.Surface
import java.lang.Double.max
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicReference
// An intermedia surface that downscales the input surface to the destination surface. Note that
// this is all done in software, so it is not very efficient and is measured at about 13ms per frame
// for the Samsung SM-S901U and ~13ms for Motorola Moto E6. However it is fast enough given that the
// capture rate is ~1 fps.
class SurfaceDownscaler(
private val srcWidth: Int, private val srcHeight: Int,
private val dstWidth: Int, private val dstHeight: Int,
private val fps: Float) {
companion object {
private const val TAG = "SurfaceDownscaler"
private const val LOG_PERFORMANCE = true
}
var inputSurface: Surface
fun setDestinationSurface(surface: Surface?) {
synchronized(surfaceLock) {
surfaceDestination = surface
}
}
fun pause() {
paused.set(true)
}
fun resume() {
paused.set(false)
}
private val paused = AtomicBoolean(false)
private val surfaceLock = Any()
private var surfaceDestination: Surface? = null
private var imageFormat: Int = PixelFormat.RGBA_8888
private var imageReader: ImageReader
private val recentImage: AtomicReference<Image?> = AtomicReference(null)
private val timerAllowImage = TimerIntervalThreaded(TAG, this::onTick)
private lateinit var largeBitmap: Bitmap
private val paint = Paint(Paint.ANTI_ALIAS_FLAG)
private val imageListener: ImageReader.OnImageAvailableListener = ImageReader.OnImageAvailableListener {
val image: Image? = it.acquireLatestImage()
if (image == null) {
return@OnImageAvailableListener
}
val prevImage: Image? = recentImage.getAndSet(image)
if (prevImage != null) {
prevImage.close()
}
}
init {
assert(srcWidth > 0)
assert(srcHeight > 0)
assert(dstWidth > 0)
assert(dstHeight > 0)
@SuppressLint("WrongConstant")
imageReader = ImageReader.newInstance(srcWidth, srcHeight, imageFormat, 3)
imageReader.setOnImageAvailableListener(imageListener, null)
timerAllowImage.start()
inputSurface = imageReader.surface
paint.isFilterBitmap = true
}
fun close() {
timerAllowImage.stop()
imageReader.close()
recentImage.getAndSet(null)?.close()
if (::largeBitmap.isInitialized) {
largeBitmap.recycle()
}
}
private fun drawImageAndRelease(image: Image) {
val startTime = System.currentTimeMillis()
val planes = image.planes
val pixelStride = planes[0].pixelStride
val rowStride = planes[0].rowStride
val rowPadding: Int = rowStride - pixelStride * image.width
if (!::largeBitmap.isInitialized) {
largeBitmap = Bitmap.createBitmap(
srcWidth + rowPadding / pixelStride, srcHeight,
Bitmap.Config.ARGB_8888
)
}
largeBitmap.copyPixelsFromBuffer(planes[0].buffer)
image.close()
synchronized(surfaceLock) {
val surface = surfaceDestination
if (surface != null) {
val canvas = surface.lockHardwareCanvas()
val dstRect = Rect(0, 0, dstWidth, dstHeight)
canvas.drawBitmap(largeBitmap, null, dstRect, paint)
surface.unlockCanvasAndPost(canvas)
}
}
val duration = System.currentTimeMillis() - startTime
if (LOG_PERFORMANCE) {
Log.d(TAG, "drawImageAndRelease duration: $duration")
}
}
private fun onTick(): Long {
val startTimeAll = System.currentTimeMillis()
val image = recentImage.getAndSet(null)
if (image != null) {
if (paused.get()) {
image.close()
} else {
drawImageAndRelease(image)
}
}
val endTime = System.currentTimeMillis()
val duration = endTime - startTimeAll
val sleepTime = max(0.0, (1000.0 / fps) - duration.toDouble())
if (LOG_PERFORMANCE) {
Log.d(TAG, "duration: $duration")
}
return sleepTime.toLong()
}
}
