From bacd4713da719038b9259e824615aa364f2edeb2 Mon Sep 17 00:00:00 2001 From: jpark37 Date: Fri, 3 Jun 2022 21:28:08 -0700 Subject: [PATCH] libobs: Clear low bits when writing P010 Don't want to rely on consumer to ignore those bits. --- libobs/data/format_conversion.effect | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/libobs/data/format_conversion.effect b/libobs/data/format_conversion.effect index 9fa6b8cc2..bf0ea2420 100644 --- a/libobs/data/format_conversion.effect +++ b/libobs/data/format_conversion.effect @@ -202,32 +202,32 @@ float PS_Y(FragPos frag_in) : TARGET return y; } -float PS_PQ_Y_709_2020(FragPos frag_in) : TARGET +float PS_P010_PQ_Y_709_2020(FragPos frag_in) : TARGET { float3 rgb = image.Load(int3(frag_in.pos.xy, 0)).rgb * sdr_white_nits_over_maximum; rgb = rec709_to_rec2020(rgb); rgb = linear_to_st2084(rgb); float y = dot(color_vec0.xyz, rgb) + color_vec0.w; - y = (65472. / 65535.) * y + (32. / 65535.); // set up truncation to 10 bits + y = floor(saturate(y) * 1023. + 0.5) * (64. / 65535.); return y; } -float PS_HLG_Y_709_2020(FragPos frag_in) : TARGET +float PS_P010_HLG_Y_709_2020(FragPos frag_in) : TARGET { float3 rgb = image.Load(int3(frag_in.pos.xy, 0)).rgb * sdr_white_nits_over_maximum; rgb = rec709_to_rec2020(rgb); rgb = linear_to_hlg(rgb, hdr_lw); float y = dot(color_vec0.xyz, rgb) + color_vec0.w; - y = (65472. / 65535.) * y + (32. / 65535.); // set up truncation to 10 bits + y = floor(saturate(y) * 1023. + 0.5) * (64. / 65535.); return y; } -float PS_SRGB_Y(FragPos frag_in) : TARGET +float PS_P010_SRGB_Y(FragPos frag_in) : TARGET { float3 rgb = image.Load(int3(frag_in.pos.xy, 0)).rgb; rgb = srgb_linear_to_nonlinear(rgb); float y = dot(color_vec0.xyz, rgb) + color_vec0.w; - y = (65472. / 65535.) * y + (32. / 65535.); // set up truncation to 10 bits + y = floor(saturate(y) * 1023. + 0.5) * (64. / 65535.); return y; } @@ -267,7 +267,7 @@ float2 PS_UV_Wide(FragTexWide frag_in) : TARGET return float2(u, v); } -float2 PS_PQ_UV_709_2020_WideWide(FragTexWideWide frag_in) : TARGET +float2 PS_P010_PQ_UV_709_2020_WideWide(FragTexWideWide frag_in) : TARGET { float3 rgb_topleft = image.Sample(def_sampler, frag_in.uuvv.xz).rgb; float3 rgb_topright = image.Sample(def_sampler, frag_in.uuvv.yz).rgb; @@ -279,11 +279,11 @@ float2 PS_PQ_UV_709_2020_WideWide(FragTexWideWide frag_in) : TARGET float u = dot(color_vec1.xyz, rgb) + color_vec1.w; float v = dot(color_vec2.xyz, rgb) + color_vec2.w; float2 uv = float2(u, v); - uv = (65472. / 65535.) * uv + (32. / 65535.); // set up truncation to 10 bits + uv = floor(saturate(uv) * 1023. + 0.5) * (64. / 65535.); return uv; } -float2 PS_HLG_UV_709_2020_WideWide(FragTexWideWide frag_in) : TARGET +float2 PS_P010_HLG_UV_709_2020_WideWide(FragTexWideWide frag_in) : TARGET { float3 rgb_topleft = image.Sample(def_sampler, frag_in.uuvv.xz).rgb; float3 rgb_topright = image.Sample(def_sampler, frag_in.uuvv.yz).rgb; @@ -295,11 +295,11 @@ float2 PS_HLG_UV_709_2020_WideWide(FragTexWideWide frag_in) : TARGET float u = dot(color_vec1.xyz, rgb) + color_vec1.w; float v = dot(color_vec2.xyz, rgb) + color_vec2.w; float2 uv = float2(u, v); - uv = (65472. / 65535.) * uv + (32. / 65535.); // set up truncation to 10 bits + uv = floor(saturate(uv) * 1023. + 0.5) * (64. / 65535.); return uv; } -float2 PS_SRGB_UV_WideWide(FragTexWideWide frag_in) : TARGET +float2 PS_P010_SRGB_UV_WideWide(FragTexWideWide frag_in) : TARGET { float3 rgb_topleft = image.Sample(def_sampler, frag_in.uuvv.xz).rgb; float3 rgb_topright = image.Sample(def_sampler, frag_in.uuvv.yz).rgb; @@ -310,7 +310,7 @@ float2 PS_SRGB_UV_WideWide(FragTexWideWide frag_in) : TARGET float u = dot(color_vec1.xyz, rgb) + color_vec1.w; float v = dot(color_vec2.xyz, rgb) + color_vec2.w; float2 uv = float2(u, v); - uv = (65472. / 65535.) * uv + (32. / 65535.); // set up truncation to 10 bits + uv = floor(saturate(uv) * 1023. + 0.5) * (64. / 65535.); return uv; } @@ -878,7 +878,7 @@ technique P010_PQ_Y pass { vertex_shader = VSPos(id); - pixel_shader = PS_PQ_Y_709_2020(frag_in); + pixel_shader = PS_P010_PQ_Y_709_2020(frag_in); } } @@ -887,7 +887,7 @@ technique P010_HLG_Y pass { vertex_shader = VSPos(id); - pixel_shader = PS_HLG_Y_709_2020(frag_in); + pixel_shader = PS_P010_HLG_Y_709_2020(frag_in); } } @@ -896,7 +896,7 @@ technique P010_SRGB_Y pass { vertex_shader = VSPos(id); - pixel_shader = PS_SRGB_Y(frag_in); + pixel_shader = PS_P010_SRGB_Y(frag_in); } } @@ -905,7 +905,7 @@ technique P010_PQ_UV pass { vertex_shader = VSTexPos_TopLeft(id); - pixel_shader = PS_PQ_UV_709_2020_WideWide(frag_in); + pixel_shader = PS_P010_PQ_UV_709_2020_WideWide(frag_in); } } @@ -914,7 +914,7 @@ technique P010_HLG_UV pass { vertex_shader = VSTexPos_TopLeft(id); - pixel_shader = PS_HLG_UV_709_2020_WideWide(frag_in); + pixel_shader = PS_P010_HLG_UV_709_2020_WideWide(frag_in); } } @@ -923,7 +923,7 @@ technique P010_SRGB_UV pass { vertex_shader = VSTexPos_TopLeft(id); - pixel_shader = PS_SRGB_UV_WideWide(frag_in); + pixel_shader = PS_P010_SRGB_UV_WideWide(frag_in); } }