diff --git a/build.gradle.kts b/build.gradle.kts index 6c2519c..271c375 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -13,6 +13,7 @@ plugins { val deployNative = (findProperty("deploy.native") as String?)?.toBoolean() ?: true val deployKotlin = (findProperty("deploy.kotlin") as String?)?.toBoolean() ?: true +val skikoEGL = (findProperty("skiko.egl") as String?)?.toBoolean() ?: false dependencies { // Note, if you develop a library, you should use compose.desktop.common. @@ -29,11 +30,13 @@ dependencies { implementation(project(":native")) } implementation(kotlin("reflect")) -// implementation(libs.bundles.skiko) { -// version { -// strictly(libs.skiko.awt.runtime.linux.x64.get().version!!) -// } -// } + if (skikoEGL) { + implementation(libs.bundles.skiko) { + version { + strictly(libs.skiko.awt.runtime.linux.x64.get().version!!) + } + } + } testImplementation(libs.bundles.kotest) testImplementation(libs.mockk) diff --git a/gradle.properties b/gradle.properties index 53094af..84fe48d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,3 +2,4 @@ org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 kotlin.code.style=official kotlin.version=2.0.0 compose.version=1.6.11 +skiko.egl=false diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index ac1d4e7..906b05b 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -66,7 +66,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") src/cpp/platform/linux/Errors.cpp src/cpp/platform/linux/VaapiDeviceContext.cpp src/cpp/platform/linux/VaapiYuvToRgbConversion.cpp - src/cpp/platform/linux/VaapiYuvToRgbConversion.hpp src/cpp/platform/linux/VAEGLInteropImage.cpp src/cpp/platform/linux/VAEGLInteropImage.hpp src/cpp/platform/linux/VAGLXInteropImage.cpp diff --git a/native/src/cpp/demux/Stream.cpp b/native/src/cpp/demux/Stream.cpp index 9adacfb..5be4cbd 100644 --- a/native/src/cpp/demux/Stream.cpp +++ b/native/src/cpp/demux/Stream.cpp @@ -45,6 +45,15 @@ JNIEXPORT jlong JNICALL Java_dev_silenium_compose_av_demux_StreamKt_bitRateN( return stream->codecpar->bit_rate; } +JNIEXPORT jint JNICALL Java_dev_silenium_compose_av_demux_StreamKt_formatN( + JNIEnv *env, + jobject thiz, + const jlong context +) { + const auto stream = reinterpret_cast(context); + return stream->codecpar->format; +} + JNIEXPORT jobject JNICALL Java_dev_silenium_compose_av_demux_StreamKt_avgFrameRateN( JNIEnv *env, jobject thiz, diff --git a/native/src/cpp/platform/linux/VAEGLRenderInterop.cpp b/native/src/cpp/platform/linux/VAEGLRenderInterop.cpp index 4ebfd68..7ba9d09 100644 --- a/native/src/cpp/platform/linux/VAEGLRenderInterop.cpp +++ b/native/src/cpp/platform/linux/VAEGLRenderInterop.cpp @@ -6,62 +6,51 @@ #include "helper/errors.hpp" #include "render/GLInteropImage.hpp" -#include -#include #include #include -#include -#include -#include -#include +#include #include +#include +#include #include +#include +#include +#include #include #include "helper/va.hpp" -typedef void (EGLAPIENTRYP PFNEGLIMAGETARGETTEXTURE2DOESPROC)(EGLenum target, void *image); +typedef void(EGLAPIENTRYP PFNEGLIMAGETARGETTEXTURE2DOESPROC)(EGLenum target, void *image); extern "C" { +#include #include #include -#include } - extern "C" { -JNIEXPORT jlong JNICALL -Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_getVADisplayN( - JNIEnv *env, jobject thiz, const jlong frame) { - const auto avFrame = reinterpret_cast(frame); - const auto deviceCtx = reinterpret_cast(avFrame->hw_frames_ctx->data)->device_ctx; - const auto vaContext = static_cast(deviceCtx->hwctx); - return reinterpret_cast(vaContext->display); -} - -std::map > > planeFractions{ - {AV_PIX_FMT_NV12, {{0, {1, 1}}, {1, {2, 2}}}}, - {AV_PIX_FMT_P010LE, {{0, {1, 1}}, {1, {2, 2}}}}, - {AV_PIX_FMT_P010BE, {{0, {1, 1}}, {1, {2, 2}}}}, - {AV_PIX_FMT_YUV420P, {{0, {1, 1}}, {1, {2, 2}}, {2, {2, 2}}}}, +std::map>> planeFractions{ + {AV_PIX_FMT_NV12, {{0, {1, 1}}, {1, {2, 2}}}}, + {AV_PIX_FMT_P010LE, {{0, {1, 1}}, {1, {2, 2}}}}, + {AV_PIX_FMT_P010BE, {{0, {1, 1}}, {1, {2, 2}}}}, + {AV_PIX_FMT_YUV420P, {{0, {1, 1}}, {1, {2, 2}}, {2, {2, 2}}}}, {AV_PIX_FMT_YUV420P10LE, {{0, {1, 1}}, {1, {2, 2}}, {2, {2, 2}}}}, {AV_PIX_FMT_YUV420P10BE, {{0, {1, 1}}, {1, {2, 2}}, {2, {2, 2}}}}, - {AV_PIX_FMT_YUV422P, {{0, {1, 1}}, {1, {2, 1}}, {2, {2, 1}}}}, - {AV_PIX_FMT_YUV444P, {{0, {1, 1}}, {1, {1, 1}}, {2, {1, 1}}}}, + {AV_PIX_FMT_YUV422P, {{0, {1, 1}}, {1, {2, 1}}, {2, {2, 1}}}}, + {AV_PIX_FMT_YUV444P, {{0, {1, 1}}, {1, {1, 1}}, {2, {1, 1}}}}, }; JNIEXPORT jobject JNICALL Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_mapN(JNIEnv *env, jobject thiz, - const jint pixelFormat_, - const jlong vaSurface_, const jlong vaDisplay_, - const jlong eglDisplay_) { - const auto pixelFormat = static_cast(pixelFormat_); - const auto vaDisplay = reinterpret_cast(vaDisplay_); - const auto vaSurface = static_cast(vaSurface_); + const jlong frame_, + const jlong eglDisplay_) { + const auto *frame = reinterpret_cast(frame_); + const auto pixelFormat = static_cast(frame->format); + const auto deviceCtx = reinterpret_cast(frame->hw_frames_ctx->data)->device_ctx; + const auto vaContext = static_cast(deviceCtx->hwctx); + const auto vaDisplay = vaContext->display; + const VASurfaceID vaSurface = reinterpret_cast(frame->data[3]); const auto eglDisplay = reinterpret_cast(eglDisplay_); - // std::cout << "VASurface: " << vaSurface << std::endl; - // std::cout << "VADisplay: " << vaDisplay << std::endl; - // std::cout << "EGLDisplay: " << eglDisplay << std::endl; const auto eglCreateImageKHR = getFunc("eglCreateImageKHR"); if (!eglCreateImageKHR) { @@ -141,11 +130,9 @@ Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_mapN(JNIEnv *en EGL_DMA_BUF_PLANE0_FD_EXT, fd, EGL_DMA_BUF_PLANE0_OFFSET_EXT, static_cast(offset[0]), EGL_DMA_BUF_PLANE0_PITCH_EXT, static_cast(pitch[0]), - EGL_NONE - }; + EGL_NONE}; EGLImageKHR eglImage = eglCreateImageKHR(eglDisplay, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, attribs); - // std::cout << "eglImage: " << eglImage << std::endl; long error = eglGetError(); if (eglImage == EGL_NO_IMAGE_KHR || error != EGL_SUCCESS) { closeDrm(drm); @@ -160,7 +147,6 @@ Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_mapN(JNIEnv *en glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, eglImage); - // std::cout << "bound egl image to texture" << std::endl; error = glGetError(); if (error != GL_NO_ERROR) { std::cerr << "Failed to bind egl image to texture: " << error << std::endl; @@ -180,7 +166,6 @@ Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_mapN(JNIEnv *en textures.emplace_back(texture); swizzles.emplace_back(channels); } - // std::cout << "eglVASurface: " << eglVASurface << std::endl; closeDrm(drm); glBindTexture(GL_TEXTURE_2D, prevTexture); diff --git a/native/src/cpp/platform/linux/VAGLXInteropImage.cpp b/native/src/cpp/platform/linux/VAGLXInteropImage.cpp index a7dfc51..0c8b8be 100644 --- a/native/src/cpp/platform/linux/VAGLXInteropImage.cpp +++ b/native/src/cpp/platform/linux/VAGLXInteropImage.cpp @@ -8,18 +8,12 @@ #include VAGLXInteropImage::VAGLXInteropImage( - VADisplay display, - void *glxSurface, unsigned int texture, Swizzles swizzles) - : display(display), glxSurface(glxSurface), texture({texture}), swizzles({swizzles}) { + : texture({texture}), swizzles({swizzles}) { } VAGLXInteropImage::~VAGLXInteropImage() { - if (glxSurface != nullptr) { - vaDestroySurfaceGLX(display, glxSurface); - } - // vaDestroySurfaces(display, &surface, 1); glDeleteTextures(1, &texture[0]); } diff --git a/native/src/cpp/platform/linux/VAGLXInteropImage.hpp b/native/src/cpp/platform/linux/VAGLXInteropImage.hpp index 7f8c5fe..0337328 100644 --- a/native/src/cpp/platform/linux/VAGLXInteropImage.hpp +++ b/native/src/cpp/platform/linux/VAGLXInteropImage.hpp @@ -12,8 +12,6 @@ class VAGLXInteropImage final : public GLInteropImage { public: VAGLXInteropImage( - VADisplay display, - void *glxSurfaces, unsigned int textures, Swizzles swizzles); @@ -32,9 +30,6 @@ class VAGLXInteropImage final : public GLInteropImage { [[nodiscard]] const std::vector &planeSwizzles() const override; private: - VADisplay display{nullptr}; - VASurfaceID surface{0}; - void *glxSurface{}; std::vector texture{}; std::vector swizzles{}; }; diff --git a/native/src/cpp/platform/linux/VAGLXRenderInterop.cpp b/native/src/cpp/platform/linux/VAGLXRenderInterop.cpp index 095569d..68e5f74 100644 --- a/native/src/cpp/platform/linux/VAGLXRenderInterop.cpp +++ b/native/src/cpp/platform/linux/VAGLXRenderInterop.cpp @@ -4,134 +4,50 @@ #include "VAGLXInteropImage.hpp" #include "helper/errors.hpp" -#include "render/GLInteropImage.hpp" -#include -#include #include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "helper/drm_mapping.hpp" -#include "helper/va.hpp" extern "C" { -#include #include #include -#include } extern "C" { -JNIEXPORT jlong JNICALL -Java_dev_silenium_compose_av_platform_linux_VAGLXRenderInteropKt_getVADisplayN( - JNIEnv *env, jobject thiz, const jlong frame) { - const auto avFrame = reinterpret_cast(frame); - const auto deviceCtx = reinterpret_cast(avFrame->hw_frames_ctx->data)->device_ctx; - const auto vaContext = static_cast(deviceCtx->hwctx); - return reinterpret_cast(vaContext->display); -} - JNIEXPORT jobject JNICALL -Java_dev_silenium_compose_av_platform_linux_VAGLXRenderInteropKt_mapN(JNIEnv *env, jobject thiz, - const jlong vaSurface_, const jlong vaDisplay_, - const jlong frame_, const jlong codecContext_) { - const auto vaDisplay = reinterpret_cast(vaDisplay_); - const auto vaSurface = static_cast(vaSurface_); +Java_dev_silenium_compose_av_platform_linux_VAGLXRenderInteropKt_mapN(JNIEnv *env, jobject thiz, const jlong frame_) { const auto frame = reinterpret_cast(frame_); - std::cout << "VA Surface: " << vaSurface << std::endl; - auto ret = vaSyncSurface(vaDisplay, vaSurface); - if (ret != VA_STATUS_SUCCESS) { - return vaResultFailure(env, "vaSyncSurface", ret); + auto swFrame = av_frame_alloc(); + if (swFrame == nullptr) { + return avResultFailure(env, "allocating software frame", AVERROR(ENOMEM)); } - // struct VAOpenGLVTable { - // PFNGLXCREATEPIXMAPPROC glx_create_pixmap; - // PFNGLXDESTROYPIXMAPPROC glx_destroy_pixmap; - // PFNGLXBINDTEXIMAGEEXTPROC glx_bind_tex_image; - // PFNGLXRELEASETEXIMAGEEXTPROC glx_release_tex_image; - // PFNGLGENFRAMEBUFFERSEXTPROC gl_gen_framebuffers; - // PFNGLDELETEFRAMEBUFFERSEXTPROC gl_delete_framebuffers; - // PFNGLBINDFRAMEBUFFEREXTPROC gl_bind_framebuffer; - // PFNGLGENRENDERBUFFERSEXTPROC gl_gen_renderbuffers; - // PFNGLDELETERENDERBUFFERSEXTPROC gl_delete_renderbuffers; - // PFNGLBINDRENDERBUFFEREXTPROC gl_bind_renderbuffer; - // PFNGLRENDERBUFFERSTORAGEEXTPROC gl_renderbuffer_storage; - // PFNGLFRAMEBUFFERRENDERBUFFEREXTPROC gl_framebuffer_renderbuffer; - // PFNGLFRAMEBUFFERTEXTURE2DEXTPROC gl_framebuffer_texture_2d; - // PFNGLCHECKFRAMEBUFFERSTATUSEXTPROC gl_check_framebuffer_status; - // }; - // - // struct VADriverContextGLX { - // struct VADriverVTableGLX vtable; - // struct VAOpenGLVTable gl_vtable; - // unsigned int is_initialized : 1; - // }; - // - auto displayContext = static_cast(vaDisplay); - auto driverContext = static_cast(displayContext->pDriverContext); - // auto glxContext = static_cast(driverContext->glx); - // std::cout << "glx initialized: " << glxContext->is_initialized << std::endl; - - // auto pixmap = XCreatePixmap( - // glXGetCurrentDisplay(), - // XRootWindow(glXGetCurrentDisplay(), XDefaultScreen(glXGetCurrentDisplay())), - // frame->width, - // frame->height, - // 24 - // ); - std::cout << "Drawable: " << glXGetCurrentDrawable() << std::endl; - ret = vaPutSurface(vaDisplay, vaSurface, glXGetCurrentDrawable(), - 0, 0, frame->width, frame->height, - 0, 0, frame->width, frame->height, - nullptr, 0, 0); - - // TODO: Copy from https://github.com/intel/libva/blob/master/va/glx/va_glx_impl.c#L1050 - // TODO: Try va surface -> EGL image -> render to fbo with bound egl image wrapped pixmap -> bind as texture in glx context - - // std::cout << "Pixmap: " << pixmap << std::endl; - if (ret != VA_STATUS_SUCCESS) { - // XFreePixmap(glXGetCurrentDisplay(), pixmap); - return vaResultFailure(env, "vaPutSurface", ret); + auto ret = av_hwframe_map(swFrame, frame, AV_HWFRAME_MAP_READ | AV_HWFRAME_MAP_DIRECT); + if (ret < 0) { + ret = av_hwframe_map(swFrame, frame, AV_HWFRAME_MAP_READ); + if (ret < 0) { + ret = av_hwframe_transfer_data(swFrame, frame, 0); + } + } + if (ret < 0) { + av_frame_free(&swFrame); + return avResultFailure(env, "mapping frame", ret); } - // if (pixmap != None) { - // XFreePixmap(glXGetCurrentDisplay(), pixmap); - // } - void *glxSurface{}; GLuint texture{}; glGenTextures(1, &texture); glBindTexture(GL_TEXTURE_2D, texture); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WIDTH, frame->width); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_HEIGHT, frame->height); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_BORDER, 0); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_INTERNAL_FORMAT, GL_RGBA); - ret = vaCreateSurfaceGLX(vaDisplay, GL_TEXTURE_2D, texture, &glxSurface); - if (ret != VA_STATUS_SUCCESS) { - return vaResultFailure(env, "vaCreateSurfaceGLX", ret); - } + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, swFrame->width, swFrame->height, 0, GL_RGBA, GL_UNSIGNED_BYTE, + swFrame->data[0]); - ret = vaCopySurfaceGLX(vaDisplay, glxSurface, vaSurface, 0); - if (ret != VA_STATUS_SUCCESS) { - return vaResultFailure(env, "vaCopySurfaceGLX", ret); - } + av_frame_free(&swFrame); - const auto interopImage = new VAGLXInteropImage(vaDisplay, glxSurface, texture, {}); + const auto interopImage = new VAGLXInteropImage(texture, {}); return resultSuccess(env, reinterpret_cast(interopImage)); } diff --git a/native/src/cpp/platform/linux/VaapiDeviceContext.cpp b/native/src/cpp/platform/linux/VaapiDeviceContext.cpp index 7a2528b..facf568 100644 --- a/native/src/cpp/platform/linux/VaapiDeviceContext.cpp +++ b/native/src/cpp/platform/linux/VaapiDeviceContext.cpp @@ -7,6 +7,7 @@ #include #include "helper/errors.hpp" +#include #include #include @@ -34,12 +35,18 @@ JNIEXPORT jobject JNICALL Java_dev_silenium_compose_av_platform_linux_VaapiDevic return avResultFailure(env, "getting VA display", AVERROR_UNKNOWN); } + int majorVersion, minorVersion; + if (vaInitialize(vaDisplay, &majorVersion, &minorVersion) != VA_STATUS_SUCCESS) { + return avResultFailure(env, "initializing VA display", AVERROR_UNKNOWN); + } + std::cout << "VA-API version: " << majorVersion << "." << minorVersion << std::endl; + auto deviceRef = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI); if (deviceRef == nullptr) { return avResultFailure(env, "allocating hw device context", AVERROR(ENOMEM)); } - auto vaapi = static_cast(reinterpret_cast(deviceRef->data)->hwctx); + const auto vaapi = static_cast(reinterpret_cast(deviceRef->data)->hwctx); vaapi->display = vaDisplay; auto ret = av_hwdevice_ctx_init(deviceRef); diff --git a/native/src/cpp/platform/linux/VaapiYuvToRgbConversion.cpp b/native/src/cpp/platform/linux/VaapiYuvToRgbConversion.cpp index f5fc220..002cee7 100644 --- a/native/src/cpp/platform/linux/VaapiYuvToRgbConversion.cpp +++ b/native/src/cpp/platform/linux/VaapiYuvToRgbConversion.cpp @@ -2,7 +2,6 @@ // Created by silenium-dev on 8/1/24. // -#include "VaapiYuvToRgbConversion.hpp" #include "helper/errors.hpp" #include "helper/rationals.hpp" #include @@ -89,6 +88,8 @@ JNIEXPORT jobject JNICALL Java_dev_silenium_compose_av_platform_linux_VaapiYuvTo AVFilterInOut *filterOut{nullptr}; auto ret = avfilter_graph_parse2(filterGraph, filterString, &filterIn, &filterOut); if (ret < 0) { + av_buffer_unref(&inputFramesRef); + av_buffer_unref(&outputFramesRef); avfilter_graph_free(&filterGraph); return avResultFailure(env, "parse filter graph", ret); } @@ -106,6 +107,8 @@ JNIEXPORT jobject JNICALL Java_dev_silenium_compose_av_platform_linux_VaapiYuvTo std::cout << "In link: " << inLink << std::endl; inLink->hw_frames_ctx = av_buffer_ref(inputFramesRef); AVFilterLink *outLink = scale->outputs[0]; + std::cout << "Out link: " << outLink << std::endl; + outLink->hw_frames_ctx = av_buffer_ref(outputFramesRef); AVBufferSrcParameters srcParams{}; srcParams.format = AV_PIX_FMT_VAAPI; @@ -118,14 +121,21 @@ JNIEXPORT jobject JNICALL Java_dev_silenium_compose_av_platform_linux_VaapiYuvTo ret = av_buffersrc_parameters_set(bufferSrc, &srcParams); if (ret < 0) { av_buffer_unref(&srcParams.hw_frames_ctx); + av_buffer_unref(&inputFramesRef); + av_buffer_unref(&outputFramesRef); + av_buffer_unref(&outLink->hw_frames_ctx); + av_buffer_unref(&inLink->hw_frames_ctx); avfilter_graph_free(&filterGraph); return avResultFailure(env, "set buffer source parameters", ret); } - outLink->hw_frames_ctx = av_buffer_ref(outputFramesRef); ret = avfilter_graph_config(filterGraph, nullptr); if (ret < 0) { av_buffer_unref(&srcParams.hw_frames_ctx); + av_buffer_unref(&inputFramesRef); + av_buffer_unref(&outputFramesRef); + av_buffer_unref(&outLink->hw_frames_ctx); + av_buffer_unref(&inLink->hw_frames_ctx); avfilter_graph_free(&filterGraph); return avResultFailure(env, "config filter graph", ret); } diff --git a/native/src/cpp/platform/linux/VaapiYuvToRgbConversion.hpp b/native/src/cpp/platform/linux/VaapiYuvToRgbConversion.hpp deleted file mode 100644 index eb97758..0000000 --- a/native/src/cpp/platform/linux/VaapiYuvToRgbConversion.hpp +++ /dev/null @@ -1,14 +0,0 @@ -// -// Created by silenium-dev on 8/1/24. -// - -#ifndef COMPOSE_AV_VAAPIYUVTORGBCONVERSION_HPP -#define COMPOSE_AV_VAAPIYUVTORGBCONVERSION_HPP - -#include - -extern "C" { -JNIEXPORT jobject JNICALL Java_dev_silenium_compose_av_VaapiYuvToRgbConversion_createN(JNIEnv *env, jclass clazz, jlong _deviceRef, jlong _inputFrame); -}; - -#endif//COMPOSE_AV_VAAPIYUVTORGBCONVERSION_HPP diff --git a/src/main/kotlin/dev/silenium/compose/av/demux/Stream.kt b/src/main/kotlin/dev/silenium/compose/av/demux/Stream.kt index f047131..7f5bd87 100644 --- a/src/main/kotlin/dev/silenium/compose/av/demux/Stream.kt +++ b/src/main/kotlin/dev/silenium/compose/av/demux/Stream.kt @@ -18,6 +18,7 @@ data class Stream(val index: Int, val type: AVMediaType, override val nativePoin val duration: Long by lazy { durationN(nativePointer.address) } val bitRate: Long by lazy { bitRateN(nativePointer.address) } val avgFrameRate: Rational by lazy { avgFrameRateN(nativePointer.address) } + val format: Int by lazy { formatN(nativePointer.address) } } private external fun indexN(pointer: Long): Int @@ -27,3 +28,4 @@ private external fun timeBaseN(pointer: Long): Rational private external fun durationN(pointer: Long): Long private external fun bitRateN(pointer: Long): Long private external fun avgFrameRateN(pointer: Long): Rational +private external fun formatN(pointer: Long): Int diff --git a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAEGLRenderInterop.kt b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAEGLRenderInterop.kt index 364ad27..51714b2 100644 --- a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAEGLRenderInterop.kt +++ b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAEGLRenderInterop.kt @@ -7,33 +7,30 @@ import dev.silenium.compose.av.render.GLRenderInterop import dev.silenium.compose.av.util.Natives import org.lwjgl.egl.EGL15 +/** + * Maps a VAAPI frame to a GLInteropImage in an EGL context. + * + * **Note:** zero-copy implementation: (VAAPI -> GL), + * but only supported when using patched Skiko in Compose for Desktop. + */ class VAEGLRenderInterop( override val decoder: VaapiDecoder, private val eglDisplay: Long = EGL15.eglGetCurrentDisplay(), -) : GLRenderInterop { - init { - Natives.ensureLoaded() - } - - override fun map(frame: Frame): Result = runCatching { -// val vaDisplay = decoder.vaDisplay - val vaDisplay = getVADisplayN(frame.nativePointer.address) - val vaSurface = frame.rawData[3] -// println("VA Surface: 0x${vaSurface.toHexString()}") -// println("VA Display: 0x${vaDisplay.toHexString()}") - return mapN(frame.swFormat!!.id, vaSurface, vaDisplay, eglDisplay) +) : GLRenderInterop() { + override fun mapImpl(frame: Frame): Result = runCatching { + return mapN(frame.nativePointer.address, eglDisplay) .map { GLInteropImage(frame, it) } } override fun isSupported(frame: Frame): Boolean { return frame.isHW && frame.format == AVPixelFormat.AV_PIX_FMT_VAAPI } + + companion object { + init { + Natives.ensureLoaded() + } + } } -private external fun getVADisplayN(frame: Long): Long -private external fun mapN( - format: Int, - vaSurface: Long, - vaDisplay: Long, - eglDisplay: Long -): Result +private external fun mapN(frame: Long, eglDisplay: Long): Result diff --git a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAGLXRenderInterop.kt b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAGLXRenderInterop.kt index 49c0c94..2afc244 100644 --- a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAGLXRenderInterop.kt +++ b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VAGLXRenderInterop.kt @@ -5,38 +5,27 @@ import dev.silenium.compose.av.data.Frame import dev.silenium.compose.av.render.GLInteropImage import dev.silenium.compose.av.render.GLRenderInterop import dev.silenium.compose.av.util.Natives -import org.slf4j.LoggerFactory - -class VAGLXRenderInterop(override val decoder: VaapiDecoder) : GLRenderInterop { - init { - Natives.ensureLoaded() - log.error("VAGLXRenderInterop is not working yet") - } +/** + * Maps a VAAPI frame to a GLInteropImage in a GLX context. + * + * **Note:** Involves two copies: (VAAPI -> CPU -> GL) and is not as efficient as [VAEGLRenderInterop], + * but the only supported option on default Compose for Desktop. + */ +class VAGLXRenderInterop(override val decoder: VaapiDecoder) : GLRenderInterop() { override fun isSupported(frame: Frame): Boolean { return frame.isHW && frame.format == AVPixelFormat.AV_PIX_FMT_VAAPI } - override fun map(frame: Frame): Result { - val vaDisplay = getVADisplayN(frame.nativePointer.address) - val vaSurface = frame.rawData[3] - return mapN( - vaSurface, - vaDisplay, - frame.nativePointer.address, - decoder.nativePointer.address - ).map { GLInteropImage(frame, it) } + override fun mapImpl(frame: Frame): Result { + return mapN(frame.nativePointer.address).map { GLInteropImage(frame, it) } } companion object { - private val log = LoggerFactory.getLogger(VAGLXRenderInterop::class.java) + init { + Natives.ensureLoaded() + } } } -private external fun getVADisplayN(frame: Long): Long -private external fun mapN( - vaSurface: Long, - vaDisplay: Long, - frame: Long, - codecContext: Long, -): Result +private external fun mapN(frame: Long): Result diff --git a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDecoder.kt b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDecoder.kt index 8128f80..ca70e47 100644 --- a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDecoder.kt +++ b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDecoder.kt @@ -11,7 +11,10 @@ class VaapiDecoder(stream: Stream, val context: VaapiDeviceContext) : Decoder VAEGLRenderInterop(this) + is VaapiDeviceContext.GLX -> VAGLXRenderInterop(this) + } companion object { init { diff --git a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDeviceContext.kt b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDeviceContext.kt index 008ca1e..c8852ca 100644 --- a/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDeviceContext.kt +++ b/src/main/kotlin/dev/silenium/compose/av/platform/linux/VaapiDeviceContext.kt @@ -5,6 +5,7 @@ import dev.silenium.compose.av.data.NativePointer import dev.silenium.compose.av.data.asNativePointer import dev.silenium.compose.av.util.Natives import dev.silenium.compose.av.util.destroyAVBufferN +import dev.silenium.compose.gl.context.EGLContext import dev.silenium.compose.gl.context.GLXContext sealed class VaapiDeviceContext(override val nativePointer: NativePointer) : NativeCleanable { @@ -23,6 +24,12 @@ sealed class VaapiDeviceContext(override val nativePointer: NativePointer) : Nat init { Natives.ensureLoaded() } + + fun detect() = when { + EGLContext.fromCurrent() != null -> DRM("/dev/dri/renderD128") + GLXContext.fromCurrent() != null -> GLX() + else -> error("No context current, please specify explicit context") + } } } diff --git a/src/main/kotlin/dev/silenium/compose/av/player/Player.kt b/src/main/kotlin/dev/silenium/compose/av/player/Player.kt index bf47693..9e29ede 100644 --- a/src/main/kotlin/dev/silenium/compose/av/player/Player.kt +++ b/src/main/kotlin/dev/silenium/compose/av/player/Player.kt @@ -12,11 +12,14 @@ import androidx.compose.runtime.remember import androidx.compose.ui.Modifier import androidx.compose.ui.unit.dp import dev.silenium.compose.av.data.AVMediaType +import dev.silenium.compose.av.data.AVPixelFormat import dev.silenium.compose.av.data.AVPixelFormat.* import dev.silenium.compose.av.data.Frame +import dev.silenium.compose.av.data.fromId import dev.silenium.compose.av.demux.FileDemuxer import dev.silenium.compose.av.platform.linux.VaapiDecoder import dev.silenium.compose.av.platform.linux.VaapiDeviceContext +import dev.silenium.compose.av.platform.linux.VaapiYuvToRgbConversion import dev.silenium.compose.av.render.GLInteropImage import dev.silenium.compose.av.render.GLRenderInterop import dev.silenium.compose.av.util.Resources.loadTextFromClasspath @@ -33,6 +36,7 @@ import kotlin.time.Duration.Companion.milliseconds class VideoPlayer(path: Path) : AutoCloseable { val demuxer = FileDemuxer(path) private var decoder: VaapiDecoder? = null + private var conversion: VaapiYuvToRgbConversion? = null private var deviceContext: VaapiDeviceContext? = null // private val decoder = SoftwareDecoder(demuxer.streams.first { it.type == AVMediaType.AVMEDIA_TYPE_VIDEO }) @@ -53,7 +57,17 @@ class VideoPlayer(path: Path) : AutoCloseable { while (decoder.submit(packet).isFailure) delay(10.milliseconds) while (isActive) { val frame = decoder.receive().getOrNull() ?: break - frames.send(frame) + if (conversion == null) { + conversion = VaapiYuvToRgbConversion(deviceContext!!, frame) + } + conversion!!.submit(frame).onFailure { + println("Failed to filter frame: $it") + }.getOrNull() ?: continue + frame.close() + val converted = conversion!!.receive().onFailure { + println("Failed to receive frame: $it") + }.getOrNull() ?: continue + frames.send(converted) } } } @@ -64,14 +78,14 @@ class VideoPlayer(path: Path) : AutoCloseable { val fragmentShader = glCreateShader(GL_FRAGMENT_SHADER) glShaderSource(vertexShader, loadTextFromClasspath("shaders/video.vert")) glCompileShader(vertexShader) - glGetShaderInfoLog(vertexShader).let(::println) + glGetShaderInfoLog(vertexShader).takeIf(String::isNotBlank)?.let(::println) glAttachShader(shaderProgram, vertexShader) glShaderSource(fragmentShader, loadTextFromClasspath("shaders/video.frag")) glCompileShader(fragmentShader) - glGetShaderInfoLog(fragmentShader).let(::println) + glGetShaderInfoLog(fragmentShader).takeIf(String::isNotBlank)?.let(::println) glAttachShader(shaderProgram, fragmentShader) glLinkProgram(shaderProgram) - glGetProgramInfoLog(shaderProgram).let(::println) + glGetProgramInfoLog(shaderProgram).takeIf(String::isNotBlank)?.let(::println) glDeleteShader(vertexShader) glDeleteShader(fragmentShader) // Check status @@ -85,7 +99,7 @@ class VideoPlayer(path: Path) : AutoCloseable { // TODO: Get parameters from decoder glUseProgram(shaderProgram) val formatLocation = glGetUniformLocation(shaderProgram, "tex_format") - glUniform1i(formatLocation, 2) + glUniform1i(formatLocation, 1) val limitedLocation = glGetUniformLocation(shaderProgram, "limitedRange") glUniform1i(limitedLocation, 1) val texYLocation = glGetUniformLocation(shaderProgram, "tex_y") @@ -132,7 +146,7 @@ class VideoPlayer(path: Path) : AutoCloseable { private suspend fun initializeGL() { if (glInitialized) return - deviceContext = VaapiDeviceContext.GLX() + deviceContext = VaapiDeviceContext.detect() decoder = VaapiDecoder(demuxer.streams.first { it.type == AVMediaType.AVMEDIA_TYPE_VIDEO }, deviceContext!!) println("Codec: ${decoder!!.stream.codec.description}") initShader() @@ -141,7 +155,7 @@ class VideoPlayer(path: Path) : AutoCloseable { glInitialized = true } - private suspend fun renderImage(image: GLInteropImage) { + private suspend fun renderImage(image: GLInteropImage, hdr: Boolean = false) { glClearColor(0f, 0f, 0f, 1f) glClear(GL_COLOR_BUFFER_BIT) @@ -157,13 +171,7 @@ class VideoPlayer(path: Path) : AutoCloseable { } val hdrLocation = glGetUniformLocation(shaderProgram, "enableHDR") - if ((image.frame.swFormat ?: image.frame.format) in setOf( - AV_PIX_FMT_P010LE, - AV_PIX_FMT_P010BE, - AV_PIX_FMT_YUV420P10LE, - AV_PIX_FMT_YUV420P10BE - ) - ) { + if (hdr) { glUniform1i(hdrLocation, 1) } else { glUniform1i(hdrLocation, 0) @@ -185,7 +193,16 @@ class VideoPlayer(path: Path) : AutoCloseable { println("Failed to map frame: $it") return }.use { - renderImage(it) + val format = fromId(decoder!!.stream.format) + val hdr = when (format) { + AV_PIX_FMT_P010LE, + AV_PIX_FMT_P010BE, + AV_PIX_FMT_YUV420P10LE, + AV_PIX_FMT_YUV420P10BE -> true + + else -> false + } + renderImage(it, hdr) } } } @@ -193,7 +210,8 @@ class VideoPlayer(path: Path) : AutoCloseable { override fun close() { runBlocking { decodeJob.cancelAndJoin() } demuxer.close() - decoder!!.close() + decoder?.close() + conversion?.close() } } diff --git a/src/main/kotlin/dev/silenium/compose/av/render/GLRenderInterop.kt b/src/main/kotlin/dev/silenium/compose/av/render/GLRenderInterop.kt index 95c2af4..b54161d 100644 --- a/src/main/kotlin/dev/silenium/compose/av/render/GLRenderInterop.kt +++ b/src/main/kotlin/dev/silenium/compose/av/render/GLRenderInterop.kt @@ -2,9 +2,37 @@ package dev.silenium.compose.av.render import dev.silenium.compose.av.data.Frame import dev.silenium.compose.av.decode.Decoder +import dev.silenium.compose.gl.surface.RollingWindowStatistics +import kotlinx.datetime.Clock +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import kotlin.time.Duration +import kotlin.time.Duration.Companion.nanoseconds +import kotlin.time.Duration.Companion.seconds -interface GLRenderInterop> { - val decoder: D - fun isSupported(frame: Frame): Boolean - fun map(frame: Frame): Result +abstract class GLRenderInterop> { + protected val log: Logger by lazy { LoggerFactory.getLogger(javaClass) } + + private var stats = RollingWindowStatistics() + private var lastStatsPrint = Clock.System.now() + + abstract val decoder: D + abstract fun isSupported(frame: Frame): Boolean + protected abstract fun mapImpl(frame: Frame): Result + + fun map(frame: Frame): Result = runCatching { + if (!isSupported(frame)) error("Unsupported frame: $frame") + val start = System.nanoTime() + mapImpl(frame).getOrThrow().also { + val end = System.nanoTime() + stats = stats.add(end, (end - start).nanoseconds) + val now = Clock.System.now() + if (now - lastStatsPrint > 1.seconds) { + log.trace("map time min=${stats.frameTimes.min.floatMs}ms, max=${stats.frameTimes.max.floatMs}ms, avg=${stats.frameTimes.average.floatMs}ms, median=${stats.frameTimes.median.floatMs}ms") + lastStatsPrint = now + } + } + } } + +private val Duration.floatMs: Float get() = inWholeMicroseconds.toFloat() / 1000.0f diff --git a/src/main/kotlin/dev/silenium/compose/av/render/SoftwareGLRenderInterop.kt b/src/main/kotlin/dev/silenium/compose/av/render/SoftwareGLRenderInterop.kt index 502be50..c3ae0f5 100644 --- a/src/main/kotlin/dev/silenium/compose/av/render/SoftwareGLRenderInterop.kt +++ b/src/main/kotlin/dev/silenium/compose/av/render/SoftwareGLRenderInterop.kt @@ -1,23 +1,14 @@ package dev.silenium.compose.av.render -import dev.silenium.compose.av.data.AVPixelFormat import dev.silenium.compose.av.data.Frame import dev.silenium.compose.av.decode.SoftwareDecoder -class SoftwareGLRenderInterop(override val decoder: SoftwareDecoder) : GLRenderInterop { +class SoftwareGLRenderInterop(override val decoder: SoftwareDecoder) : GLRenderInterop() { override fun isSupported(frame: Frame): Boolean { return !frame.isHW } - private fun planeFractions(pixelFormat: AVPixelFormat): Map> = when (pixelFormat) { - AVPixelFormat.AV_PIX_FMT_YUV420P10LE, - AVPixelFormat.AV_PIX_FMT_YUV420P10BE, - AVPixelFormat.AV_PIX_FMT_YUV420P -> mapOf(0 to (1 to 1), 1 to (2 to 2), 2 to (2 to 2)) - - else -> error("Unsupported pixel format: $pixelFormat") - } - - override fun map(frame: Frame): Result = + override fun mapImpl(frame: Frame): Result = mapN(frame.nativePointer.address).map { GLInteropImage(frame, it) } } diff --git a/src/test/kotlin/dev/silenium/compose/av/Main.kt b/src/test/kotlin/dev/silenium/compose/av/Main.kt index 5dd6a5b..a95cec2 100644 --- a/src/test/kotlin/dev/silenium/compose/av/Main.kt +++ b/src/test/kotlin/dev/silenium/compose/av/Main.kt @@ -17,7 +17,7 @@ fun App() { MaterialTheme { val file = remember { val videoFile = Files.createTempFile("video", ".webm") - FileDemuxer::class.java.classLoader.getResourceAsStream("1080p.webm").use { + FileDemuxer::class.java.classLoader.getResourceAsStream("4K_HDR.webm").use { videoFile.outputStream().use(it::copyTo) } videoFile.apply { toFile().deleteOnExit() } diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml index 74599de..af5b8c1 100644 --- a/src/test/resources/logback.xml +++ b/src/test/resources/logback.xml @@ -4,7 +4,7 @@ %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n - +