obs-qsv11: Implement texture encoder on Linux

Implements the texture allocator for QSV using VA-API surfaces and
implement encode_texture2 for using OpenGL textures directly.

plugins/obs-qsv11/QSV_Encoder.cpp

@@ -106,7 +106,7 @@ qsv_t *qsv_encoder_open(qsv_param_t *pParams, enum qsv_codec codec)
 			WARN_ERR(MFX_ERR_NOT_FOUND,
 				 "Specified object/item/sync point not found.");
 			WARN_ERR(MFX_ERR_MEMORY_ALLOC,
-				 "Gailed to allocate memory");
+				 "Failed to allocate memory");
 			WARN_ERR(MFX_ERR_LOCK_MEMORY,
 				 "failed to lock the memory block "
 				 "(external allocator).");
@@ -212,14 +212,14 @@ int qsv_encoder_encode(qsv_t *pContext, uint64_t ts, uint8_t *pDataY,
 		return -1;
 }
 
-int qsv_encoder_encode_tex(qsv_t *pContext, uint64_t ts, uint32_t tex_handle,
+int qsv_encoder_encode_tex(qsv_t *pContext, uint64_t ts, void *tex,
 			   uint64_t lock_key, uint64_t *next_key,
 			   mfxBitstream **pBS)
 {
 	QSV_Encoder_Internal *pEncoder = (QSV_Encoder_Internal *)pContext;
 	mfxStatus sts = MFX_ERR_NONE;
 
-	sts = pEncoder->Encode_tex(ts, tex_handle, lock_key, next_key, pBS);
+	sts = pEncoder->Encode_tex(ts, tex, lock_key, next_key, pBS);
 
 	if (sts == MFX_ERR_NONE)
 		return 0;

plugins/obs-qsv11/QSV_Encoder.h

@@ -162,7 +162,7 @@ void qsv_encoder_add_roi(qsv_t *, const struct obs_encoder_roi *roi);
 void qsv_encoder_clear_roi(qsv_t *pContext);
 int qsv_encoder_encode(qsv_t *, uint64_t, uint8_t *, uint8_t *, uint32_t,
 		       uint32_t, mfxBitstream **pBS);
-int qsv_encoder_encode_tex(qsv_t *, uint64_t, uint32_t, uint64_t, uint64_t *,
+int qsv_encoder_encode_tex(qsv_t *, uint64_t, void *, uint64_t, uint64_t *,
 			   mfxBitstream **pBS);
 int qsv_encoder_headers(qsv_t *, uint8_t **pSPS, uint8_t **pPPS,
 			uint16_t *pnSPS, uint16_t *pnPPS);

plugins/obs-qsv11/QSV_Encoder_Internal.cpp

@@ -68,7 +68,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define info(format, ...) do_log(LOG_INFO, format, ##__VA_ARGS__)
 #define debug(format, ...) do_log(LOG_DEBUG, format, ##__VA_ARGS__)
 
-mfxHDL QSV_Encoder_Internal::g_DX_Handle = NULL;
+mfxHDL QSV_Encoder_Internal::g_GFX_Handle = NULL;
 mfxU16 QSV_Encoder_Internal::g_numEncodersOpen = 0;
 
 QSV_Encoder_Internal::QSV_Encoder_Internal(mfxVersion &version)
@@ -109,7 +109,7 @@ QSV_Encoder_Internal::QSV_Encoder_Internal(mfxVersion &version)
 		cfg, (const mfxU8 *)"mfxImplDescription.AccelerationMode",
 		tempImpl);
 #else
-	m_bUseTexAlloc = false;
+	m_bUseTexAlloc = true;
 	tempImpl.Type = MFX_VARIANT_TYPE_U32;
 	tempImpl.Data.U32 = MFX_ACCEL_MODE_VIA_VAAPI;
 	MFXSetConfigFilterProperty(
@@ -140,18 +140,13 @@ mfxStatus QSV_Encoder_Internal::Open(qsv_param_t *pParams, enum qsv_codec codec)
 {
 	mfxStatus sts = MFX_ERR_NONE;
 
-#if defined(_WIN32)
-	if (m_bUseD3D11)
-		// Use D3D11 surface
+	if (m_bUseD3D11 | m_bUseTexAlloc)
+		// Use texture surface
 		sts = Initialize(m_ver, &m_session, &m_mfxAllocator,
-				 &g_DX_Handle, false, codec, &m_sessionData);
+				 &g_GFX_Handle, false, codec, &m_sessionData);
 	else
-		sts = Initialize(m_ver, &m_session, NULL, NULL, NULL, codec,
+		sts = Initialize(m_ver, &m_session, NULL, NULL, false, codec,
 				 &m_sessionData);
-#else
-	sts = Initialize(m_ver, &m_session, NULL, NULL, false, codec,
-			 &m_sessionData);
-#endif
 
 	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
 
@@ -801,7 +796,7 @@ mfxStatus QSV_Encoder_Internal::Encode(uint64_t ts, uint8_t *pDataY,
 	return sts;
 }
 
-mfxStatus QSV_Encoder_Internal::Encode_tex(uint64_t ts, uint32_t tex_handle,
+mfxStatus QSV_Encoder_Internal::Encode_tex(uint64_t ts, void *tex,
 					   uint64_t lock_key,
 					   uint64_t *next_key,
 					   mfxBitstream **pBS)
@@ -838,7 +833,7 @@ mfxStatus QSV_Encoder_Internal::Encode_tex(uint64_t ts, uint32_t tex_handle,
 	if (m_bUseTexAlloc) {
 		// mfxU64 isn't consistent with stdint, requiring a cast to be multi-platform.
 		sts = simple_copytex(m_mfxAllocator.pthis, pSurface->Data.MemId,
-				     tex_handle, lock_key, (mfxU64 *)next_key);
+				     tex, lock_key, (mfxU64 *)next_key);
 		MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
 	}
 
@@ -923,7 +918,7 @@ mfxStatus QSV_Encoder_Internal::ClearData()
 
 	if ((m_bUseTexAlloc) && (g_numEncodersOpen <= 0)) {
 		Release();
-		g_DX_Handle = NULL;
+		g_GFX_Handle = NULL;
 	}
 	MFXVideoENCODE_Close(m_session);
 	ReleaseSessionData(m_sessionData);

plugins/obs-qsv11/QSV_Encoder_Internal.h

@@ -74,9 +74,8 @@ public:
 	mfxStatus Encode(uint64_t ts, uint8_t *pDataY, uint8_t *pDataUV,
 			 uint32_t strideY, uint32_t strideUV,
 			 mfxBitstream **pBS);
-	mfxStatus Encode_tex(uint64_t ts, uint32_t tex_handle,
-			     uint64_t lock_key, uint64_t *next_key,
-			     mfxBitstream **pBS);
+	mfxStatus Encode_tex(uint64_t ts, void *tex, uint64_t lock_key,
+			     uint64_t *next_key, mfxBitstream **pBS);
 	mfxStatus ClearData();
 	mfxStatus Reset(qsv_param_t *pParams, enum qsv_codec codec);
 	mfxStatus ReconfigureEncoder();
@@ -134,7 +133,7 @@ private:
 	bool m_bUseTexAlloc;
 	static mfxU16 g_numEncodersOpen;
 	static mfxHDL
-		g_DX_Handle; // we only want one handle for all instances to use;
+		g_GFX_Handle; // we only want one handle for all instances to use;
 
 	mfxEncodeCtrl m_ctrl;
 	mfxExtEncoderROI m_roi;

plugins/obs-qsv11/common_directx11.cpp

@@ -1,5 +1,7 @@
 #include "common_directx11.h"
 
+#include <obs.h>
+#include <obs-encoder.h>
 #include <map>
 
 ID3D11Device *g_pD3D11Device = nullptr;
@@ -433,10 +435,11 @@ mfxStatus simple_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
 	return MFX_ERR_NONE;
 }
 
-mfxStatus simple_copytex(mfxHDL pthis, mfxMemId mid, mfxU32 tex_handle,
-			 mfxU64 lock_key, mfxU64 *next_key)
+mfxStatus simple_copytex(mfxHDL pthis, mfxMemId mid, void *tex, mfxU64 lock_key,
+			 mfxU64 *next_key)
 {
 	pthis; // To suppress warning for this unused parameter
+	struct encoder_texture *ptex = (struct encoder_texture *)tex;
 
 	CustomMemId *memId = (CustomMemId *)mid;
 	ID3D11Texture2D *pSurface = (ID3D11Texture2D *)memId->memId;
@@ -445,7 +448,7 @@ mfxStatus simple_copytex(mfxHDL pthis, mfxMemId mid, mfxU32 tex_handle,
 	ID3D11Texture2D *input_tex;
 	HRESULT hr;
 
-	hr = g_pD3D11Device->OpenSharedResource((HANDLE)(uintptr_t)tex_handle,
+	hr = g_pD3D11Device->OpenSharedResource((HANDLE)(uintptr_t)ptex->handle,
 						IID_ID3D11Texture2D,
 						(void **)&input_tex);
 	if (FAILED(hr)) {

plugins/obs-qsv11/common_utils.h

@@ -117,8 +117,8 @@ mfxStatus simple_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr);
 mfxStatus simple_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr);
 mfxStatus simple_gethdl(mfxHDL pthis, mfxMemId mid, mfxHDL *handle);
 mfxStatus simple_free(mfxHDL pthis, mfxFrameAllocResponse *response);
-mfxStatus simple_copytex(mfxHDL pthis, mfxMemId mid, mfxU32 tex_handle,
-			 mfxU64 lock_key, mfxU64 *next_key);
+mfxStatus simple_copytex(mfxHDL pthis, mfxMemId mid, void *tex, mfxU64 lock_key,
+			 mfxU64 *next_key);
 
 // =================================================================
 // Utility functions, not directly tied to VPL functionality

plugins/obs-qsv11/common_utils_linux.cpp

@@ -2,9 +2,9 @@
 
 #include <time.h>
 #include <cpuid.h>
-#include <util/c99defs.h>
-#include <util/dstr.h>
+
 #include <va/va_drm.h>
+#include <va/va_drmcommon.h>
 #include <va/va_str.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -14,7 +14,12 @@
 #include <dirent.h>
 
 #include <obs.h>
+#include <obs-encoder.h>
 #include <obs-nix-platform.h>
+#include <graphics/graphics.h>
+#include <util/c99defs.h>
+#include <util/dstr.h>
+#include <util/bmem.h>
 
 // Set during check_adapters to work-around VPL dispatcher not setting a VADisplay
 // for the MSDK runtime.
@@ -27,13 +32,145 @@ struct linux_data {
 	VADisplay vaDisplay;
 };
 
+#define DEVICE_MGR_TYPE MFX_HANDLE_VA_DISPLAY
+// This ends up at like 72 for 1440p@120 AV1.
+// We may end up hitting this in practice?
+constexpr int32_t MAX_ALLOCABLE_SURFACES = 128;
+
+struct surface_info {
+	VASurfaceID id;
+	int32_t width, height;
+	gs_texture_t *tex_y;
+	gs_texture_t *tex_uv;
+};
+
 mfxStatus simple_alloc(mfxHDL pthis, mfxFrameAllocRequest *request,
 		       mfxFrameAllocResponse *response)
 {
-	UNUSED_PARAMETER(pthis);
-	UNUSED_PARAMETER(request);
-	UNUSED_PARAMETER(response);
-	return MFX_ERR_UNSUPPORTED;
+	if (request->Type & (MFX_MEMTYPE_SYSTEM_MEMORY |
+			     MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET))
+		return MFX_ERR_UNSUPPORTED;
+
+	response->mids = (mfxMemId *)nullptr;
+	response->NumFrameActual = 0;
+
+	mfxSession *session = (mfxSession *)pthis;
+	VADisplay display;
+	mfxStatus sts =
+		MFXVideoCORE_GetHandle(*session, DEVICE_MGR_TYPE, &display);
+	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
+
+	// https://ffmpeg.org/doxygen/5.1/hwcontext__vaapi_8c_source.html#l00109
+	// though earlier comments suggest the driver ignores rt_format so we could choose whatever.
+	unsigned int rt_format;
+	int32_t pix_format;
+	switch (request->Info.FourCC) {
+	case MFX_FOURCC_P010:
+		rt_format = VA_RT_FORMAT_YUV420_10;
+		pix_format = VA_FOURCC_P010;
+		break;
+	case MFX_FOURCC_NV12:
+	default:
+		rt_format = VA_RT_FORMAT_YUV420;
+		pix_format = VA_FOURCC_NV12;
+		break;
+	}
+
+	int num_attrs = 2;
+	VASurfaceAttrib attrs[2] = {
+		{
+			.type = VASurfaceAttribMemoryType,
+			.flags = VA_SURFACE_ATTRIB_SETTABLE,
+			.value =
+				{
+					.type = VAGenericValueTypeInteger,
+					.value =
+						{.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2},
+				},
+		},
+		{
+			.type = VASurfaceAttribPixelFormat,
+			.flags = VA_SURFACE_ATTRIB_SETTABLE,
+			.value =
+				{
+					.type = VAGenericValueTypeInteger,
+					.value = {.i = (int)pix_format},
+				},
+		}};
+
+	unsigned int num_surfaces = request->NumFrameSuggested;
+	VASurfaceID temp_surfaces[MAX_ALLOCABLE_SURFACES] = {0};
+	assert(num_surfaces < MAX_ALLOCABLE_SURFACES);
+	VAStatus vasts;
+	if ((vasts = vaCreateSurfaces(display, rt_format, request->Info.Width,
+				      request->Info.Height, temp_surfaces,
+				      num_surfaces, attrs, num_attrs)) !=
+	    VA_STATUS_SUCCESS) {
+		blog(LOG_ERROR, "failed to create surfaces: %d", vasts);
+		return MFX_ERR_MEMORY_ALLOC;
+	}
+
+	// Follow the FFmpeg trick and stuff our pointer at the end.
+	mfxMemId *mids =
+		(mfxMemId *)bmalloc(sizeof(mfxMemId) * num_surfaces + 1);
+	struct surface_info *surfaces = (struct surface_info *)bmalloc(
+		sizeof(struct surface_info) * num_surfaces);
+
+	mids[num_surfaces] = surfaces; // stuff it
+	for (uint64_t i = 0; i < num_surfaces; i++) {
+		surfaces[i].id = temp_surfaces[i];
+		surfaces[i].width = request->Info.Width;
+		surfaces[i].height = request->Info.Height;
+		mids[i] = &surfaces[i];
+
+		VADRMPRIMESurfaceDescriptor surfDesc = {0};
+		if (vaExportSurfaceHandle(display, surfaces[i].id,
+					  VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
+					  VA_EXPORT_SURFACE_READ_WRITE,
+					  &surfDesc) != VA_STATUS_SUCCESS)
+			return MFX_ERR_MEMORY_ALLOC;
+
+		obs_enter_graphics();
+		// TODO: P010 format support
+		assert(surfDesc.num_objects == 1);
+		int fds[4] = {0};
+		uint32_t strides[4] = {0};
+		uint32_t offsets[4] = {0};
+		uint64_t modifiers[4] = {0};
+		fds[0] =
+			surfDesc.objects[surfDesc.layers[0].object_index[0]].fd;
+		fds[1] =
+			surfDesc.objects[surfDesc.layers[1].object_index[0]].fd;
+		strides[0] = surfDesc.layers[0].pitch[0];
+		strides[1] = surfDesc.layers[1].pitch[0];
+		offsets[0] = surfDesc.layers[0].offset[0];
+		offsets[1] = surfDesc.layers[1].offset[0];
+		modifiers[0] =
+			surfDesc.objects[surfDesc.layers[0].object_index[0]]
+				.drm_format_modifier;
+		modifiers[1] =
+			surfDesc.objects[surfDesc.layers[1].object_index[0]]
+				.drm_format_modifier;
+
+		surfaces[i].tex_y = gs_texture_create_from_dmabuf(
+			surfDesc.width, surfDesc.height,
+			surfDesc.layers[0].drm_format, GS_R8, 1, fds, strides,
+			offsets, modifiers);
+		surfaces[i].tex_uv = gs_texture_create_from_dmabuf(
+			surfDesc.width / 2, surfDesc.height,
+			surfDesc.layers[1].drm_format, GS_R8G8, 1, fds + 1,
+			strides + 1, offsets + 1, modifiers + 1);
+		obs_leave_graphics();
+
+		close(surfDesc.objects[surfDesc.layers[0].object_index[0]].fd);
+		if (!surfaces[i].tex_y || !surfaces[i].tex_uv) {
+			return MFX_ERR_MEMORY_ALLOC;
+		}
+	}
+
+	response->mids = (mfxMemId *)mids;
+	response->NumFrameActual = num_surfaces;
+	return MFX_ERR_NONE;
 }
 
 mfxStatus simple_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
@@ -55,33 +192,78 @@ mfxStatus simple_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr)
 mfxStatus simple_gethdl(mfxHDL pthis, mfxMemId mid, mfxHDL *handle)
 {
 	UNUSED_PARAMETER(pthis);
-	UNUSED_PARAMETER(mid);
-	UNUSED_PARAMETER(handle);
-	return MFX_ERR_UNSUPPORTED;
+	if (NULL == handle)
+		return MFX_ERR_INVALID_HANDLE;
+
+	// Seemingly undocumented, but Pair format defined by
+	// oneVPL-intel-gpu-intel-onevpl-23.1.0/_studio/mfx_lib/encode_hw/av1/linux/base/av1ehw_base_va_packer_lin.cpp
+	// https://github.com/intel/vpl-gpu-rt/blob/4170dd9fa1ea319dda81b6189616ecc9b178a321/_studio/shared/src/libmfx_core_vaapi.cpp#L1464
+	mfxHDLPair *pPair = (mfxHDLPair *)handle;
+
+	// first must be a pointer to a VASurfaceID and will be dereferenced by
+	// the driver.
+	pPair->first = &((struct surface_info *)mid)->id;
+	pPair->second = 0;
+
+	return MFX_ERR_NONE;
 }
 
 mfxStatus simple_free(mfxHDL pthis, mfxFrameAllocResponse *response)
 {
-	UNUSED_PARAMETER(pthis);
-	UNUSED_PARAMETER(response);
-	return MFX_ERR_UNSUPPORTED;
+	if (response->mids == nullptr || response->NumFrameActual == 0)
+		return MFX_ERR_NONE;
+
+	mfxSession *session = (mfxSession *)pthis;
+	VADisplay display;
+	mfxStatus sts =
+		MFXVideoCORE_GetHandle(*session, DEVICE_MGR_TYPE, &display);
+	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
+
+	struct surface_info *surfs =
+		(struct surface_info *)response->mids[response->NumFrameActual];
+	VASurfaceID temp_surfaces[MAX_ALLOCABLE_SURFACES] = {0};
+	obs_enter_graphics();
+	for (int i = 0; i < response->NumFrameActual; i++) {
+		temp_surfaces[i] = *(VASurfaceID *)response->mids[i];
+		gs_texture_destroy(surfs[i].tex_y);
+		gs_texture_destroy(surfs[i].tex_uv);
+	}
+	obs_leave_graphics();
+
+	bfree(surfs);
+	bfree(response->mids);
+	if (vaDestroySurfaces(display, temp_surfaces,
+			      response->NumFrameActual) != VA_STATUS_SUCCESS)
+		return MFX_ERR_MEMORY_ALLOC;
+
+	return MFX_ERR_NONE;
 }
 
-mfxStatus simple_copytex(mfxHDL pthis, mfxMemId mid, mfxU32 tex_handle,
-			 mfxU64 lock_key, mfxU64 *next_key)
+mfxStatus simple_copytex(mfxHDL pthis, mfxMemId mid, void *tex, mfxU64 lock_key,
+			 mfxU64 *next_key)
 {
-	UNUSED_PARAMETER(pthis);
-	UNUSED_PARAMETER(mid);
-	UNUSED_PARAMETER(tex_handle);
 	UNUSED_PARAMETER(lock_key);
 	UNUSED_PARAMETER(next_key);
-	return MFX_ERR_UNSUPPORTED;
-}
 
-#if 0
-void ClearYUVSurfaceVMem(mfxMemId memId);
-void ClearRGBSurfaceVMem(mfxMemId memId);
-#endif
+	profile_start("copy_tex");
+
+	mfxSession *session = (mfxSession *)pthis;
+	VADisplay display;
+	mfxStatus sts =
+		MFXVideoCORE_GetHandle(*session, DEVICE_MGR_TYPE, &display);
+	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
+
+	struct encoder_texture *ptex = (struct encoder_texture *)tex;
+	struct surface_info *surf = (struct surface_info *)mid;
+
+	obs_enter_graphics();
+	gs_copy_texture(surf->tex_y, ptex->tex[0]);
+	gs_copy_texture(surf->tex_uv, ptex->tex[1]);
+	obs_leave_graphics();
+
+	profile_end("copy_tex");
+	return MFX_ERR_NONE;
+}
 
 // Initialize Intel VPL Session, device/display and memory manager
 mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
@@ -90,7 +272,6 @@ mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		     void **data)
 {
 	UNUSED_PARAMETER(ver);
-	UNUSED_PARAMETER(pmfxAllocator);
 	UNUSED_PARAMETER(deviceHandle);
 	UNUSED_PARAMETER(bCreateSharedHandles);
 	mfxStatus sts = MFX_ERR_NONE;
@@ -117,12 +298,24 @@ mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		impl);
 
 	int fd = -1;
-	if (codec == QSV_CODEC_AVC && default_h264_device)
-		fd = open(default_h264_device, O_RDWR);
-	if (codec == QSV_CODEC_HEVC && default_hevc_device)
-		fd = open(default_hevc_device, O_RDWR);
-	if (codec == QSV_CODEC_AV1 && default_av1_device)
-		fd = open(default_av1_device, O_RDWR);
+	if (pmfxAllocator) {
+		// TODO: This is broken and we need the ovi adapter to be
+		// correct for checks earlier in encoder_create to fallback
+		// properly.
+		char device_path[128];
+		obs_video_info ovi;
+		obs_get_video_info(&ovi);
+		// eglQueryDeviceStringEXT( device, EGL_DRM_DEVICE_FILE_EXT);
+		sprintf(device_path, "/dev/dri/renderD%d", 128 + ovi.adapter);
+		fd = open(device_path, O_RDWR);
+	} else {
+		if (codec == QSV_CODEC_AVC && default_h264_device)
+			fd = open(default_h264_device, O_RDWR);
+		else if (codec == QSV_CODEC_HEVC && default_hevc_device)
+			fd = open(default_hevc_device, O_RDWR);
+		else if (codec == QSV_CODEC_AV1 && default_av1_device)
+			fd = open(default_av1_device, O_RDWR);
+	}
 	if (fd < 0) {
 		blog(LOG_ERROR, "Failed to open device '%s'",
 		     default_h264_device);
@@ -152,10 +345,22 @@ mfxStatus Initialize(mfxVersion ver, mfxSession *pSession,
 		return MFX_ERR_DEVICE_FAILED;
 	}
 
-	sts = MFXVideoCORE_SetHandle(*pSession, MFX_HANDLE_VA_DISPLAY,
-				     vaDisplay);
+	sts = MFXVideoCORE_SetHandle(*pSession, DEVICE_MGR_TYPE, vaDisplay);
 	MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
 
+	if (pmfxAllocator) {
+		// Allow us to access the session during allocation.
+		pmfxAllocator->pthis = pSession;
+		pmfxAllocator->Alloc = simple_alloc;
+		pmfxAllocator->Free = simple_free;
+		pmfxAllocator->Lock = simple_lock;
+		pmfxAllocator->Unlock = simple_unlock;
+		pmfxAllocator->GetHDL = simple_gethdl;
+
+		sts = MFXVideoCORE_SetFrameAllocator(*pSession, pmfxAllocator);
+		MSDK_CHECK_RESULT(sts, MFX_ERR_NONE, sts);
+	}
+
 	struct linux_data *d =
 		(struct linux_data *)bmalloc(sizeof(struct linux_data));
 	d->fd = fd;
@@ -187,7 +392,7 @@ double TimeDiffMsec(mfxTime tfinish, mfxTime tstart)
 {
 	UNUSED_PARAMETER(tfinish);
 	UNUSED_PARAMETER(tstart);
-	//TODO, unused so far it seems
+	//unused so far
 	return 0.0;
 }
 

plugins/obs-qsv11/obs-qsv11.c

@@ -1420,14 +1420,19 @@ static bool obs_qsv_encode(void *data, struct encoder_frame *frame,
 	return true;
 }
 
-static bool obs_qsv_encode_tex(void *data, uint32_t handle, int64_t pts,
-			       uint64_t lock_key, uint64_t *next_key,
+static bool obs_qsv_encode_tex(void *data, struct encoder_texture *tex,
+			       int64_t pts, uint64_t lock_key,
+			       uint64_t *next_key,
 			       struct encoder_packet *packet,
 			       bool *received_packet)
 {
 	struct obs_qsv *obsqsv = data;
 
-	if (handle == GS_INVALID_HANDLE) {
+#ifdef _WIN32
+	if (!tex || tex->handle == GS_INVALID_HANDLE) {
+#else
+	if (!tex || !tex->tex[0] || !tex->tex[1]) {
+#endif
 		warn("Encode failed: bad texture handle");
 		*next_key = lock_key;
 		return false;
@@ -1450,8 +1455,8 @@ static bool obs_qsv_encode_tex(void *data, uint32_t handle, int64_t pts,
 	if (obs_encoder_has_roi(obsqsv->encoder))
 		obs_qsv_setup_rois(obsqsv);
 
-	ret = qsv_encoder_encode_tex(obsqsv->context, qsvPTS, handle, lock_key,
-				     next_key, &pBS);
+	ret = qsv_encoder_encode_tex(obsqsv->context, qsvPTS, (void *)tex,
+				     lock_key, next_key, &pBS);
 
 	if (ret < 0) {
 		warn("encode failed");
@@ -1480,7 +1485,7 @@ struct obs_encoder_info obs_qsv_encoder_tex = {
 	.destroy = obs_qsv_destroy,
 	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE |
 		OBS_ENCODER_CAP_DEPRECATED,
-	.encode_texture = obs_qsv_encode_tex,
+	.encode_texture2 = obs_qsv_encode_tex,
 	.update = obs_qsv_update,
 	.get_properties = obs_qsv_props_h264,
 	.get_defaults = obs_qsv_defaults_h264_v1,
@@ -1516,7 +1521,7 @@ struct obs_encoder_info obs_qsv_encoder_tex_v2 = {
 	.destroy = obs_qsv_destroy,
 	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE |
 		OBS_ENCODER_CAP_ROI,
-	.encode_texture = obs_qsv_encode_tex,
+	.encode_texture2 = obs_qsv_encode_tex,
 	.update = obs_qsv_update,
 	.get_properties = obs_qsv_props_h264_v2,
 	.get_defaults = obs_qsv_defaults_h264_v2,
@@ -1552,7 +1557,7 @@ struct obs_encoder_info obs_qsv_av1_encoder_tex = {
 	.destroy = obs_qsv_destroy,
 	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE |
 		OBS_ENCODER_CAP_ROI,
-	.encode_texture = obs_qsv_encode_tex,
+	.encode_texture2 = obs_qsv_encode_tex,
 	.update = obs_qsv_update,
 	.get_properties = obs_qsv_props_av1,
 	.get_defaults = obs_qsv_defaults_av1,
@@ -1586,7 +1591,7 @@ struct obs_encoder_info obs_qsv_hevc_encoder_tex = {
 	.destroy = obs_qsv_destroy,
 	.caps = OBS_ENCODER_CAP_DYN_BITRATE | OBS_ENCODER_CAP_PASS_TEXTURE |
 		OBS_ENCODER_CAP_ROI,
-	.encode_texture = obs_qsv_encode_tex,
+	.encode_texture2 = obs_qsv_encode_tex,
 	.update = obs_qsv_update,
 	.get_properties = obs_qsv_props_hevc,
 	.get_defaults = obs_qsv_defaults_hevc,