Skip to content

Commit

Permalink
feat: implement vaapi glx interop with copy
Browse files Browse the repository at this point in the history
  • Loading branch information
@silenium-dev
silenium-dev committed Aug 1, 2024
1 parent 1c3c65f commit 66c594e
Show file tree
Hide file tree
Showing 21 changed files with 195 additions and 255 deletions.
13 changes: 8 additions & 5 deletions build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ plugins {

val deployNative = (findProperty("deploy.native") as String?)?.toBoolean() ?: true
val deployKotlin = (findProperty("deploy.kotlin") as String?)?.toBoolean() ?: true
val skikoEGL = (findProperty("skiko.egl") as String?)?.toBoolean() ?: false

dependencies {
// Note, if you develop a library, you should use compose.desktop.common.
Expand All @@ -29,11 +30,13 @@ dependencies {
implementation(project(":native"))
}
implementation(kotlin("reflect"))
// implementation(libs.bundles.skiko) {
// version {
// strictly(libs.skiko.awt.runtime.linux.x64.get().version!!)
// }
// }
if (skikoEGL) {
implementation(libs.bundles.skiko) {
version {
strictly(libs.skiko.awt.runtime.linux.x64.get().version!!)
}
}
}

testImplementation(libs.bundles.kotest)
testImplementation(libs.mockk)
Expand Down
1 change: 1 addition & 0 deletions gradle.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
kotlin.code.style=official
kotlin.version=2.0.0
compose.version=1.6.11
skiko.egl=false
1 change: 0 additions & 1 deletion native/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
src/cpp/platform/linux/Errors.cpp
src/cpp/platform/linux/VaapiDeviceContext.cpp
src/cpp/platform/linux/VaapiYuvToRgbConversion.cpp
src/cpp/platform/linux/VaapiYuvToRgbConversion.hpp
src/cpp/platform/linux/VAEGLInteropImage.cpp
src/cpp/platform/linux/VAEGLInteropImage.hpp
src/cpp/platform/linux/VAGLXInteropImage.cpp
Expand Down
9 changes: 9 additions & 0 deletions native/src/cpp/demux/Stream.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,15 @@ JNIEXPORT jlong JNICALL Java_dev_silenium_compose_av_demux_StreamKt_bitRateN(
return stream->codecpar->bit_rate;
}

JNIEXPORT jint JNICALL Java_dev_silenium_compose_av_demux_StreamKt_formatN(
JNIEnv *env,
jobject thiz,
const jlong context
) {
const auto stream = reinterpret_cast<AVStream *>(context);
return stream->codecpar->format;
}

JNIEXPORT jobject JNICALL Java_dev_silenium_compose_av_demux_StreamKt_avgFrameRateN(
JNIEnv *env,
jobject thiz,
Expand Down
63 changes: 24 additions & 39 deletions native/src/cpp/platform/linux/VAEGLRenderInterop.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,62 +6,51 @@
#include "helper/errors.hpp"
#include "render/GLInteropImage.hpp"

#include <jni.h>
#include <GL/gl.h>
#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <va/va.h>
#include <va/va_drmcommon.h>
#include <unistd.h>
#include <drm_fourcc.h>
#include <GL/gl.h>
#include <algorithm>
#include <drm_fourcc.h>
#include <jni.h>
#include <map>
#include <unistd.h>
#include <va/va.h>
#include <va/va_drmcommon.h>
#include <vector>

#include "helper/va.hpp"

typedef void (EGLAPIENTRYP PFNEGLIMAGETARGETTEXTURE2DOESPROC)(EGLenum target, void *image);
typedef void(EGLAPIENTRYP PFNEGLIMAGETARGETTEXTURE2DOESPROC)(EGLenum target, void *image);

extern "C" {
#include <libavutil/frame.h>
#include <libavutil/hwcontext.h>
#include <libavutil/hwcontext_vaapi.h>
#include <libavutil/frame.h>
}


extern "C" {
JNIEXPORT jlong JNICALL
Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_getVADisplayN(
JNIEnv *env, jobject thiz, const jlong frame) {
const auto avFrame = reinterpret_cast<AVFrame *>(frame);
const auto deviceCtx = reinterpret_cast<AVHWFramesContext *>(avFrame->hw_frames_ctx->data)->device_ctx;
const auto vaContext = static_cast<AVVAAPIDeviceContext *>(deviceCtx->hwctx);
return reinterpret_cast<jlong>(vaContext->display);
}

std::map<AVPixelFormat, std::map<int, std::pair<int, int> > > planeFractions{
{AV_PIX_FMT_NV12, {{0, {1, 1}}, {1, {2, 2}}}},
{AV_PIX_FMT_P010LE, {{0, {1, 1}}, {1, {2, 2}}}},
{AV_PIX_FMT_P010BE, {{0, {1, 1}}, {1, {2, 2}}}},
{AV_PIX_FMT_YUV420P, {{0, {1, 1}}, {1, {2, 2}}, {2, {2, 2}}}},
std::map<AVPixelFormat, std::map<int, std::pair<int, int>>> planeFractions{
{AV_PIX_FMT_NV12, {{0, {1, 1}}, {1, {2, 2}}}},
{AV_PIX_FMT_P010LE, {{0, {1, 1}}, {1, {2, 2}}}},
{AV_PIX_FMT_P010BE, {{0, {1, 1}}, {1, {2, 2}}}},
{AV_PIX_FMT_YUV420P, {{0, {1, 1}}, {1, {2, 2}}, {2, {2, 2}}}},
{AV_PIX_FMT_YUV420P10LE, {{0, {1, 1}}, {1, {2, 2}}, {2, {2, 2}}}},
{AV_PIX_FMT_YUV420P10BE, {{0, {1, 1}}, {1, {2, 2}}, {2, {2, 2}}}},
{AV_PIX_FMT_YUV422P, {{0, {1, 1}}, {1, {2, 1}}, {2, {2, 1}}}},
{AV_PIX_FMT_YUV444P, {{0, {1, 1}}, {1, {1, 1}}, {2, {1, 1}}}},
{AV_PIX_FMT_YUV422P, {{0, {1, 1}}, {1, {2, 1}}, {2, {2, 1}}}},
{AV_PIX_FMT_YUV444P, {{0, {1, 1}}, {1, {1, 1}}, {2, {1, 1}}}},
};

JNIEXPORT jobject JNICALL
Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_mapN(JNIEnv *env, jobject thiz,
const jint pixelFormat_,
const jlong vaSurface_, const jlong vaDisplay_,
const jlong eglDisplay_) {
const auto pixelFormat = static_cast<AVPixelFormat>(pixelFormat_);
const auto vaDisplay = reinterpret_cast<VADisplay>(vaDisplay_);
const auto vaSurface = static_cast<VASurfaceID>(vaSurface_);
const jlong frame_,
const jlong eglDisplay_) {
const auto *frame = reinterpret_cast<AVFrame *>(frame_);
const auto pixelFormat = static_cast<AVPixelFormat>(frame->format);
const auto deviceCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data)->device_ctx;
const auto vaContext = static_cast<AVVAAPIDeviceContext *>(deviceCtx->hwctx);
const auto vaDisplay = vaContext->display;
const VASurfaceID vaSurface = reinterpret_cast<intptr_t>(frame->data[3]);
const auto eglDisplay = reinterpret_cast<EGLDisplay>(eglDisplay_);
// std::cout << "VASurface: " << vaSurface << std::endl;
// std::cout << "VADisplay: " << vaDisplay << std::endl;
// std::cout << "EGLDisplay: " << eglDisplay << std::endl;

const auto eglCreateImageKHR = getFunc<PFNEGLCREATEIMAGEKHRPROC>("eglCreateImageKHR");
if (!eglCreateImageKHR) {
Expand Down Expand Up @@ -141,11 +130,9 @@ Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_mapN(JNIEnv *en
EGL_DMA_BUF_PLANE0_FD_EXT, fd,
EGL_DMA_BUF_PLANE0_OFFSET_EXT, static_cast<EGLint>(offset[0]),
EGL_DMA_BUF_PLANE0_PITCH_EXT, static_cast<EGLint>(pitch[0]),
EGL_NONE
};
EGL_NONE};

EGLImageKHR eglImage = eglCreateImageKHR(eglDisplay, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, attribs);
// std::cout << "eglImage: " << eglImage << std::endl;
long error = eglGetError();
if (eglImage == EGL_NO_IMAGE_KHR || error != EGL_SUCCESS) {
closeDrm(drm);
Expand All @@ -160,7 +147,6 @@ Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_mapN(JNIEnv *en
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, eglImage);
// std::cout << "bound egl image to texture" << std::endl;
error = glGetError();
if (error != GL_NO_ERROR) {
std::cerr << "Failed to bind egl image to texture: " << error << std::endl;
Expand All @@ -180,7 +166,6 @@ Java_dev_silenium_compose_av_platform_linux_VAEGLRenderInteropKt_mapN(JNIEnv *en
textures.emplace_back(texture);
swizzles.emplace_back(channels);
}
// std::cout << "eglVASurface: " << eglVASurface << std::endl;

closeDrm(drm);
glBindTexture(GL_TEXTURE_2D, prevTexture);
Expand Down
8 changes: 1 addition & 7 deletions native/src/cpp/platform/linux/VAGLXInteropImage.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,12 @@
#include <va/va_glx.h>

VAGLXInteropImage::VAGLXInteropImage(
VADisplay display,
void *glxSurface,
unsigned int texture,
Swizzles swizzles)
: display(display), glxSurface(glxSurface), texture({texture}), swizzles({swizzles}) {
: texture({texture}), swizzles({swizzles}) {
}

VAGLXInteropImage::~VAGLXInteropImage() {
if (glxSurface != nullptr) {
vaDestroySurfaceGLX(display, glxSurface);
}
// vaDestroySurfaces(display, &surface, 1);
glDeleteTextures(1, &texture[0]);
}

Expand Down
5 changes: 0 additions & 5 deletions native/src/cpp/platform/linux/VAGLXInteropImage.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
class VAGLXInteropImage final : public GLInteropImage {
public:
VAGLXInteropImage(
VADisplay display,
void *glxSurfaces,
unsigned int textures,
Swizzles swizzles);

Expand All @@ -32,9 +30,6 @@ class VAGLXInteropImage final : public GLInteropImage {
[[nodiscard]] const std::vector<Swizzles> &planeSwizzles() const override;

private:
VADisplay display{nullptr};
VASurfaceID surface{0};
void *glxSurface{};
std::vector<unsigned int> texture{};
std::vector<Swizzles> swizzles{};
};
Expand Down
124 changes: 20 additions & 104 deletions native/src/cpp/platform/linux/VAGLXRenderInterop.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,134 +4,50 @@

#include "VAGLXInteropImage.hpp"
#include "helper/errors.hpp"
#include "render/GLInteropImage.hpp"

#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <GL/gl.h>
#include <GL/glx.h>
#include <algorithm>
#include <drm_fourcc.h>
#include <jni.h>
#include <map>
#include <unistd.h>
#include <va/va_drmcommon.h>
#include <va/va_glx.h>
#include <va/va_x11.h>
#include <vector>
#include <va/va_backend.h>
#include <va/va_backend_glx.h>

#include "helper/drm_mapping.hpp"
#include "helper/va.hpp"

extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/frame.h>
#include <libavutil/hwcontext.h>
#include <libavutil/hwcontext_vaapi.h>
}

extern "C" {
JNIEXPORT jlong JNICALL
Java_dev_silenium_compose_av_platform_linux_VAGLXRenderInteropKt_getVADisplayN(
JNIEnv *env, jobject thiz, const jlong frame) {
const auto avFrame = reinterpret_cast<AVFrame *>(frame);
const auto deviceCtx = reinterpret_cast<AVHWFramesContext *>(avFrame->hw_frames_ctx->data)->device_ctx;
const auto vaContext = static_cast<AVVAAPIDeviceContext *>(deviceCtx->hwctx);
return reinterpret_cast<jlong>(vaContext->display);
}

JNIEXPORT jobject JNICALL
Java_dev_silenium_compose_av_platform_linux_VAGLXRenderInteropKt_mapN(JNIEnv *env, jobject thiz,
const jlong vaSurface_, const jlong vaDisplay_,
const jlong frame_, const jlong codecContext_) {
const auto vaDisplay = reinterpret_cast<VADisplay>(vaDisplay_);
const auto vaSurface = static_cast<VASurfaceID>(vaSurface_);
Java_dev_silenium_compose_av_platform_linux_VAGLXRenderInteropKt_mapN(JNIEnv *env, jobject thiz, const jlong frame_) {
const auto frame = reinterpret_cast<AVFrame *>(frame_);
std::cout << "VA Surface: " << vaSurface << std::endl;

auto ret = vaSyncSurface(vaDisplay, vaSurface);
if (ret != VA_STATUS_SUCCESS) {
return vaResultFailure(env, "vaSyncSurface", ret);
auto swFrame = av_frame_alloc();
if (swFrame == nullptr) {
return avResultFailure(env, "allocating software frame", AVERROR(ENOMEM));
}

// struct VAOpenGLVTable {
// PFNGLXCREATEPIXMAPPROC glx_create_pixmap;
// PFNGLXDESTROYPIXMAPPROC glx_destroy_pixmap;
// PFNGLXBINDTEXIMAGEEXTPROC glx_bind_tex_image;
// PFNGLXRELEASETEXIMAGEEXTPROC glx_release_tex_image;
// PFNGLGENFRAMEBUFFERSEXTPROC gl_gen_framebuffers;
// PFNGLDELETEFRAMEBUFFERSEXTPROC gl_delete_framebuffers;
// PFNGLBINDFRAMEBUFFEREXTPROC gl_bind_framebuffer;
// PFNGLGENRENDERBUFFERSEXTPROC gl_gen_renderbuffers;
// PFNGLDELETERENDERBUFFERSEXTPROC gl_delete_renderbuffers;
// PFNGLBINDRENDERBUFFEREXTPROC gl_bind_renderbuffer;
// PFNGLRENDERBUFFERSTORAGEEXTPROC gl_renderbuffer_storage;
// PFNGLFRAMEBUFFERRENDERBUFFEREXTPROC gl_framebuffer_renderbuffer;
// PFNGLFRAMEBUFFERTEXTURE2DEXTPROC gl_framebuffer_texture_2d;
// PFNGLCHECKFRAMEBUFFERSTATUSEXTPROC gl_check_framebuffer_status;
// };
//
// struct VADriverContextGLX {
// struct VADriverVTableGLX vtable;
// struct VAOpenGLVTable gl_vtable;
// unsigned int is_initialized : 1;
// };
//
auto displayContext = static_cast<VADisplayContextP>(vaDisplay);
auto driverContext = static_cast<VADriverContextP>(displayContext->pDriverContext);
// auto glxContext = static_cast<VADriverContextGLX*>(driverContext->glx);
// std::cout << "glx initialized: " << glxContext->is_initialized << std::endl;

// auto pixmap = XCreatePixmap(
// glXGetCurrentDisplay(),
// XRootWindow(glXGetCurrentDisplay(), XDefaultScreen(glXGetCurrentDisplay())),
// frame->width,
// frame->height,
// 24
// );
std::cout << "Drawable: " << glXGetCurrentDrawable() << std::endl;
ret = vaPutSurface(vaDisplay, vaSurface, glXGetCurrentDrawable(),
0, 0, frame->width, frame->height,
0, 0, frame->width, frame->height,
nullptr, 0, 0);

// TODO: Copy from https://github.com/intel/libva/blob/master/va/glx/va_glx_impl.c#L1050
// TODO: Try va surface -> EGL image -> render to fbo with bound egl image wrapped pixmap -> bind as texture in glx context

// std::cout << "Pixmap: " << pixmap << std::endl;
if (ret != VA_STATUS_SUCCESS) {
// XFreePixmap(glXGetCurrentDisplay(), pixmap);
return vaResultFailure(env, "vaPutSurface", ret);
auto ret = av_hwframe_map(swFrame, frame, AV_HWFRAME_MAP_READ | AV_HWFRAME_MAP_DIRECT);
if (ret < 0) {
ret = av_hwframe_map(swFrame, frame, AV_HWFRAME_MAP_READ);
if (ret < 0) {
ret = av_hwframe_transfer_data(swFrame, frame, 0);
}
}
if (ret < 0) {
av_frame_free(&swFrame);
return avResultFailure(env, "mapping frame", ret);
}
// if (pixmap != None) {
// XFreePixmap(glXGetCurrentDisplay(), pixmap);
// }

void *glxSurface{};
GLuint texture{};
glGenTextures(1, &texture);
glBindTexture(GL_TEXTURE_2D, texture);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WIDTH, frame->width);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_HEIGHT, frame->height);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_BORDER, 0);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_INTERNAL_FORMAT, GL_RGBA);
ret = vaCreateSurfaceGLX(vaDisplay, GL_TEXTURE_2D, texture, &glxSurface);
if (ret != VA_STATUS_SUCCESS) {
return vaResultFailure(env, "vaCreateSurfaceGLX", ret);
}
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, swFrame->width, swFrame->height, 0, GL_RGBA, GL_UNSIGNED_BYTE,
swFrame->data[0]);

ret = vaCopySurfaceGLX(vaDisplay, glxSurface, vaSurface, 0);
if (ret != VA_STATUS_SUCCESS) {
return vaResultFailure(env, "vaCopySurfaceGLX", ret);
}
av_frame_free(&swFrame);

const auto interopImage = new VAGLXInteropImage(vaDisplay, glxSurface, texture, {});
const auto interopImage = new VAGLXInteropImage(texture, {});

return resultSuccess(env, reinterpret_cast<jlong>(interopImage));
}
Expand Down
9 changes: 8 additions & 1 deletion native/src/cpp/platform/linux/VaapiDeviceContext.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <unistd.h>

#include "helper/errors.hpp"
#include <iostream>
#include <va/va_drm.h>
#include <va/va_x11.h>

Expand Down Expand Up @@ -34,12 +35,18 @@ JNIEXPORT jobject JNICALL Java_dev_silenium_compose_av_platform_linux_VaapiDevic
return avResultFailure(env, "getting VA display", AVERROR_UNKNOWN);
}

int majorVersion, minorVersion;
if (vaInitialize(vaDisplay, &majorVersion, &minorVersion) != VA_STATUS_SUCCESS) {
return avResultFailure(env, "initializing VA display", AVERROR_UNKNOWN);
}
std::cout << "VA-API version: " << majorVersion << "." << minorVersion << std::endl;

auto deviceRef = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI);
if (deviceRef == nullptr) {
return avResultFailure(env, "allocating hw device context", AVERROR(ENOMEM));
}

auto vaapi = static_cast<AVVAAPIDeviceContext *>(reinterpret_cast<AVHWDeviceContext *>(deviceRef->data)->hwctx);
const auto vaapi = static_cast<AVVAAPIDeviceContext *>(reinterpret_cast<AVHWDeviceContext *>(deviceRef->data)->hwctx);
vaapi->display = vaDisplay;

auto ret = av_hwdevice_ctx_init(deviceRef);
Expand Down
Loading

0 comments on commit 66c594e

Please sign in to comment.