obs-nvenc: Add new NVENC plugin

plugins/CMakeLists.txt

@@ -55,6 +55,7 @@ if(OBS_CMAKE_VERSION VERSION_GREATER_EQUAL 3.0.0)
   add_obs_plugin(obs-ffmpeg)
   add_obs_plugin(obs-filters)
   add_obs_plugin(obs-libfdk)
+  add_obs_plugin(obs-nvenc PLATFORMS WINDOWS LINUX)
   add_obs_plugin(obs-outputs)
   add_obs_plugin(obs-qsv11 PLATFORMS WINDOWS LINUX)
   add_obs_plugin(obs-text PLATFORMS WINDOWS)

plugins/obs-nvenc/CMakeLists.txt

@@ -0,0 +1,58 @@
+cmake_minimum_required(VERSION 3.22...3.25)
+
+option(ENABLE_NVENC "Build NVIDIA Hardware Encoder Plugin" ON)
+option(ENABLE_NVENC_FFMPEG_IDS "Register FFmpeg encoder IDs" ON)
+mark_as_advanced(ENABLE_NVENC_FFMPEG_IDS)
+
+if(NOT ENABLE_NVENC)
+  target_disable_feature(obs-nvenc "NVIDIA Hardware Encoder")
+  target_disable(obs-nvenc)
+  return()
+endif()
+
+if(NOT TARGET OBS::opts-parser)
+  add_subdirectory("${CMAKE_SOURCE_DIR}/deps/opts-parser" "${CMAKE_BINARY_DIR}/deps/opts-parser")
+endif()
+
+if(OS_LINUX AND NOT TARGET OBS::glad)
+  add_subdirectory("${CMAKE_SOURCE_DIR}/deps/glad" "${CMAKE_BINARY_DIR}/deps/glad")
+endif()
+
+find_package(FFnvcodec 12 REQUIRED)
+
+add_library(obs-nvenc MODULE)
+add_library(OBS::nvenc ALIAS obs-nvenc)
+
+add_subdirectory(obs-nvenc-test)
+
+target_sources(
+  obs-nvenc
+  PRIVATE # cmake-format: sortable
+          $<$<PLATFORM_ID:Linux>:nvenc-opengl.c>
+          $<$<PLATFORM_ID:Windows>:nvenc-d3d11.c>
+          cuda-helpers.c
+          cuda-helpers.h
+          nvenc-compat.c
+          nvenc-cuda.c
+          nvenc-helpers.c
+          nvenc-helpers.h
+          nvenc-internal.h
+          nvenc-opts-parser.c
+          nvenc-properties.c
+          nvenc.c
+          obs-nvenc.c
+          obs-nvenc.h)
+
+target_link_libraries(obs-nvenc PRIVATE OBS::libobs OBS::opts-parser FFnvcodec::FFnvcodec
+                                        $<$<PLATFORM_ID:Linux>:OBS::glad>)
+
+target_compile_definitions(obs-nvenc PRIVATE $<$<BOOL:${ENABLE_NVENC_FFMPEG_IDS}>:REGISTER_FFMPEG_IDS>)
+
+if(OS_WINDOWS)
+  configure_file(cmake/windows/obs-module.rc.in obs-nvenc.rc)
+  target_sources(obs-nvenc PRIVATE obs-nvenc.rc)
+endif()
+
+# cmake-format: off
+set_target_properties_obs(obs-nvenc PROPERTIES FOLDER plugins/obs-nvenc PREFIX "")
+# cmake-format: on

plugins/obs-nvenc/cmake/windows/obs-module.rc.in

@@ -0,0 +1,24 @@
+1 VERSIONINFO
+FILEVERSION ${OBS_VERSION_MAJOR},${OBS_VERSION_MINOR},${OBS_VERSION_PATCH},0
+BEGIN
+  BLOCK "StringFileInfo"
+  BEGIN
+    BLOCK "040904B0"
+    BEGIN
+      VALUE "CompanyName", "${OBS_COMPANY_NAME}"
+      VALUE "FileDescription", "OBS NVENC module"
+      VALUE "FileVersion", "${OBS_VERSION_CANONICAL}"
+      VALUE "ProductName", "${OBS_PRODUCT_NAME}"
+      VALUE "ProductVersion", "${OBS_VERSION_CANONICAL}"
+      VALUE "Comments", "${OBS_COMMENTS}"
+      VALUE "LegalCopyright", "${OBS_LEGAL_COPYRIGHT}"
+      VALUE "InternalName", "obs-nvenc"
+      VALUE "OriginalFilename", "obs-nvenc"
+    END
+  END
+
+  BLOCK "VarFileInfo"
+  BEGIN
+    VALUE "Translation", 0x0409, 0x04B0
+  END
+END

plugins/obs-nvenc/cuda-helpers.c

@@ -0,0 +1,168 @@
+#include "obs-nvenc.h"
+
+#include "nvenc-internal.h"
+#include "cuda-helpers.h"
+
+#include <util/platform.h>
+#include <util/threading.h>
+#include <util/config-file.h>
+#include <util/dstr.h>
+#include <util/pipe.h>
+
+static void *cuda_lib = NULL;
+static pthread_mutex_t init_mutex = PTHREAD_MUTEX_INITIALIZER;
+CudaFunctions *cu = NULL;
+
+bool load_cuda_lib(void)
+{
+#ifdef _WIN32
+	cuda_lib = os_dlopen("nvcuda.dll");
+#else
+	cuda_lib = os_dlopen("libcuda.so.1");
+#endif
+	return cuda_lib != NULL;
+}
+
+static void *load_cuda_func(const char *func)
+{
+	void *func_ptr = os_dlsym(cuda_lib, func);
+	if (!func_ptr) {
+		blog(LOG_ERROR, "[obs-nvenc] Could not load function: %s",
+		     func);
+	}
+	return func_ptr;
+}
+
+typedef struct cuda_function {
+	ptrdiff_t offset;
+	const char *name;
+} cuda_function;
+
+static const cuda_function cuda_functions[] = {
+	{offsetof(CudaFunctions, cuInit), "cuInit"},
+
+	{offsetof(CudaFunctions, cuDeviceGetCount), "cuDeviceGetCount"},
+	{offsetof(CudaFunctions, cuDeviceGet), "cuDeviceGet"},
+	{offsetof(CudaFunctions, cuDeviceGetAttribute), "cuDeviceGetAttribute"},
+
+	{offsetof(CudaFunctions, cuCtxCreate), "cuCtxCreate_v2"},
+	{offsetof(CudaFunctions, cuCtxDestroy), "cuCtxDestroy_v2"},
+	{offsetof(CudaFunctions, cuCtxPushCurrent), "cuCtxPushCurrent_v2"},
+	{offsetof(CudaFunctions, cuCtxPopCurrent), "cuCtxPopCurrent_v2"},
+
+	{offsetof(CudaFunctions, cuArray3DCreate), "cuArray3DCreate_v2"},
+	{offsetof(CudaFunctions, cuArrayDestroy), "cuArrayDestroy"},
+	{offsetof(CudaFunctions, cuMemcpy2D), "cuMemcpy2D_v2"},
+
+	{offsetof(CudaFunctions, cuGetErrorName), "cuGetErrorName"},
+	{offsetof(CudaFunctions, cuGetErrorString), "cuGetErrorString"},
+
+	{offsetof(CudaFunctions, cuMemHostRegister), "cuMemHostRegister_v2"},
+	{offsetof(CudaFunctions, cuMemHostUnregister), "cuMemHostUnregister"},
+
+#ifndef _WIN32
+	{offsetof(CudaFunctions, cuGLGetDevices), "cuGLGetDevices_v2"},
+	{offsetof(CudaFunctions, cuGraphicsGLRegisterImage),
+	 "cuGraphicsGLRegisterImage"},
+	{offsetof(CudaFunctions, cuGraphicsUnregisterResource),
+	 "cuGraphicsUnregisterResource"},
+	{offsetof(CudaFunctions, cuGraphicsMapResources),
+	 "cuGraphicsMapResources"},
+	{offsetof(CudaFunctions, cuGraphicsUnmapResources),
+	 "cuGraphicsUnmapResources"},
+	{offsetof(CudaFunctions, cuGraphicsSubResourceGetMappedArray),
+	 "cuGraphicsSubResourceGetMappedArray"},
+#endif
+};
+
+static const size_t num_cuda_funcs =
+	sizeof(cuda_functions) / sizeof(cuda_function);
+
+static bool init_cuda_internal(obs_encoder_t *encoder)
+{
+	static bool initialized = false;
+	static bool success = false;
+
+	if (initialized)
+		return success;
+	initialized = true;
+
+	if (!load_cuda_lib()) {
+		obs_encoder_set_last_error(encoder,
+					   "Loading CUDA library failed.");
+		return false;
+	}
+
+	cu = bzalloc(sizeof(CudaFunctions));
+
+	for (size_t idx = 0; idx < num_cuda_funcs; idx++) {
+		const cuda_function func = cuda_functions[idx];
+		void *fptr = load_cuda_func(func.name);
+
+		if (!fptr) {
+			blog(LOG_ERROR,
+			     "[obs-nvenc] Failed to find CUDA function: %s",
+			     func.name);
+			obs_encoder_set_last_error(
+				encoder, "Loading CUDA functions failed.");
+			return false;
+		}
+
+		*(uintptr_t *)((uintptr_t)cu + func.offset) = (uintptr_t)fptr;
+	}
+
+	success = true;
+	return true;
+}
+
+bool cuda_get_error_desc(CUresult res, const char **name, const char **desc)
+{
+	if (cu->cuGetErrorName(res, name) != CUDA_SUCCESS ||
+	    cu->cuGetErrorString(res, desc) != CUDA_SUCCESS)
+		return false;
+
+	return true;
+}
+
+bool cuda_error_check(struct nvenc_data *enc, CUresult res, const char *func,
+		      const char *call)
+{
+	if (res == CUDA_SUCCESS)
+		return true;
+
+	const char *name, *desc;
+	if (cuda_get_error_desc(res, &name, &desc)) {
+		blog(LOG_ERROR,
+		     "[obs-nvenc: '%s'] %s: CUDA call \"%s\" failed with %s (%d): %s",
+		     obs_encoder_get_name(enc->encoder), func, call, name, res,
+		     desc);
+	} else {
+		blog(LOG_ERROR,
+		     "[obs-nvenc: '%s'] %s: CUDA call \"%s\" failed with %d",
+		     obs_encoder_get_name(enc->encoder), func, call, res);
+	}
+
+	return false;
+}
+
+bool init_cuda(obs_encoder_t *encoder)
+{
+	bool success;
+
+	pthread_mutex_lock(&init_mutex);
+	success = init_cuda_internal(encoder);
+	pthread_mutex_unlock(&init_mutex);
+
+	return success;
+}
+
+void obs_cuda_load(void)
+{
+	pthread_mutex_init(&init_mutex, NULL);
+}
+
+void obs_cuda_unload(void)
+{
+	bfree(cu);
+	pthread_mutex_destroy(&init_mutex);
+}

plugins/obs-nvenc/cuda-helpers.h

@@ -0,0 +1,66 @@
+#pragma once
+
+#include <obs-module.h>
+
+#include <ffnvcodec/dynlink_cuda.h>
+
+/* Missing from FFmpeg headers */
+typedef CUresult CUDAAPI tcuMemHostRegister(void *p, size_t bytesize,
+					    unsigned int Flags);
+typedef CUresult CUDAAPI tcuMemHostUnregister(void *p);
+
+#define CUDA_ERROR_INVALID_GRAPHICS_CONTEXT 219
+#define CUDA_ARRAY3D_SURFACE_LDST 0x02
+
+typedef struct CudaFunctions {
+	tcuInit *cuInit;
+
+	tcuDeviceGetCount *cuDeviceGetCount;
+	tcuDeviceGet *cuDeviceGet;
+	tcuDeviceGetAttribute *cuDeviceGetAttribute;
+
+	tcuCtxCreate_v2 *cuCtxCreate;
+	tcuCtxDestroy_v2 *cuCtxDestroy;
+	tcuCtxPushCurrent_v2 *cuCtxPushCurrent;
+	tcuCtxPopCurrent_v2 *cuCtxPopCurrent;
+
+	tcuArray3DCreate *cuArray3DCreate;
+	tcuArrayDestroy *cuArrayDestroy;
+	tcuMemcpy2D_v2 *cuMemcpy2D;
+
+	tcuGetErrorName *cuGetErrorName;
+	tcuGetErrorString *cuGetErrorString;
+
+	tcuMemHostRegister *cuMemHostRegister;
+	tcuMemHostUnregister *cuMemHostUnregister;
+
+#ifndef _WIN32
+	tcuGLGetDevices_v2 *cuGLGetDevices;
+	tcuGraphicsGLRegisterImage *cuGraphicsGLRegisterImage;
+	tcuGraphicsUnregisterResource *cuGraphicsUnregisterResource;
+	tcuGraphicsMapResources *cuGraphicsMapResources;
+	tcuGraphicsUnmapResources *cuGraphicsUnmapResources;
+	tcuGraphicsSubResourceGetMappedArray
+		*cuGraphicsSubResourceGetMappedArray;
+#endif
+} CudaFunctions;
+
+extern CudaFunctions *cu;
+
+bool init_cuda(obs_encoder_t *encoder);
+bool cuda_get_error_desc(CUresult res, const char **name, const char **desc);
+
+struct nvenc_data;
+bool cuda_error_check(struct nvenc_data *enc, CUresult res, const char *func,
+		      const char *call);
+
+/* CUDA error handling */
+#define CU_FAILED(call)                                        \
+	if (!cuda_error_check(enc, call, __FUNCTION__, #call)) \
+		return false;
+
+#define CU_CHECK(call)                                           \
+	if (!cuda_error_check(enc, call, __FUNCTION__, #call)) { \
+		success = false;                                 \
+		goto unmap;                                      \
+	}

plugins/obs-nvenc/data/locale/en-US.ini

@@ -0,0 +1,70 @@
+RateControl="Rate Control"
+CBR="Constant Bitrate"
+VBR="Variable Bitrate"
+CQVBR="Variable Bitrate with Target Quality"
+CQP="Constant QP"
+Lossless="Lossless"
+
+Bitrate="Bitrate"
+MaxBitrate="Maximum Bitrate"
+TargetQuality="Target Quality"
+
+KeyframeIntervalSec="Keyframe interval (seconds, 0 = auto)"
+
+LookAhead="Look-ahead"
+LookAhead.ToolTip="Enables Lookahead.\n\nIf enabled, it will increase visual quality by determining a better bitrate distribution through analysis of future frames,\nat the cost of increased GPU utilization and latency."
+
+AdaptiveQuantization="Adaptive Quantization"
+AdaptiveQuantization.ToolTip="Enables Temporal/Spatial Adaptive Quantization which optimizes the use of bitrate for increased perceived visual quality,\nespecially in situations with high motion, at the cost of increased GPU utilization.\n\nFormerly known as \"Psycho-Visual Tuning\"."
+
+Preset="Preset"
+Preset.p1="P1: Fastest (Lowest Quality)"
+Preset.p2="P2: Faster (Lower Quality)"
+Preset.p3="P3: Fast (Low Quality)"
+Preset.p4="P4: Medium (Medium Quality)"
+Preset.p5="P5: Slow (Good Quality)"
+Preset.p6="P6: Slower (Better Quality)"
+Preset.p7="P7: Slowest (Best Quality)"
+
+Tuning.uhq="Ultra High Quality (slow, not recommended)"
+Tuning.hq="High Quality"
+Tuning.ll="Low Latency"
+Tuning.ull="Ultra Low Latency"
+
+Multipass="Multipass Mode"
+Multipass.disabled="Single Pass"
+Multipass.qres="Two Passes (Quarter Resolution)"
+Multipass.fullres="Two Passes (Full Resolution)"
+
+BFrames="B-Frames"
+BFrameRefMode="B-Frame as Reference"
+BframeRefMode.Disabled="Disabled"
+BframeRefMode.Each="Each"
+BframeRefMode.Middle="Middle b-frame only"
+
+SplitEncode="Split Encode"
+SplitEncode.Auto="Auto"
+SplitEncode.Disabled="Disabled"
+SplitEncode.Enabled="Two-way split"
+SplitEncode.ThreeWay="Three-way split"
+
+Opts="Custom Encoder Options"
+Opts.TT="Space-separated list of options to apply to the rate control and codec settings,\nbased their names in the nvEncodeAPI header.\ne.g. \"lookaheadDepth=16 aqStrength=4\""
+
+Error="Failed to open NVENC codec: %1"
+GenericError="Try installing the latest <a href=\"https://obsproject.com/go/nvidia-drivers\">NVIDIA driver</a> and closing other recording software that might be using NVENC such as NVIDIA ShadowPlay or Windows Game DVR."
+BadGPUIndex="You have selected GPU %1 in your output encoder settings. Set this back to 0 and try again."
+OutdatedDriver="The installed NVIDIA driver does not support this NVENC version, try <a href=\"https://obsproject.com/go/nvidia-drivers\">updating the driver</a>."
+UnsupportedDevice="NVENC Error: Unsupported device. Check that your video card <a href=\"https://obsproject.com/go/nvenc-matrix\">supports NVENC</a> and try <a href=\"https://obsproject.com/go/nvidia-drivers\">updating the driver</a>."
+TooManySessions="NVENC Error: Too many concurrent sessions. Try closing other recording software that might be using NVENC such as NVIDIA ShadowPlay or Windows Game DVR."
+CheckDrivers="Try installing the latest <a href=\"https://obsproject.com/go/nvidia-drivers\">NVIDIA driver</a>."
+
+8bitUnsupportedHdr="OBS does not support 8-bit output of Rec. 2100."
+I010Unsupported="NVENC does not support I010. Use P010 instead."
+10bitUnsupported="Cannot perform 10-bit encode on this encoder."
+16bitUnsupported="Cannot perform 16-bit encode on this encoder."
+
+# Legacy strings, to be removed once compat encoders are removed
+CQLevel="CQ Level"
+PsychoVisualTuning="Psycho Visual Tuning"
+PsychoVisualTuning.ToolTip="Enables encoder settings that optimize the use of bitrate for increased perceived visual quality,\nespecially in situations with high motion, at the cost of increased GPU utilization."