From a6dbe1501d823d0c4cbe5f84bac725a362d014fc Mon Sep 17 00:00:00 2001 From: jp9000 Date: Mon, 17 Feb 2014 09:28:27 -0700 Subject: [PATCH] Fix precision issues with new conversion shader Turns out that on some adapters, due to some sort of internal GPU precision error, fmod(x, y) can return x when x == y, wich is incorrect (and no, they were actually equal, not off due to precision errors). This would cause the shader to sample wrong coordinates on the edges sometimes. Just adding 0.1 to the x value before being put in to fmod and then flooring the result after fixes the issue. --- build/data/libobs/format_conversion.effect | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/build/data/libobs/format_conversion.effect b/build/data/libobs/format_conversion.effect index dc91edecd..45bda3315 100644 --- a/build/data/libobs/format_conversion.effect +++ b/build/data/libobs/format_conversion.effect @@ -53,6 +53,9 @@ VertInOut VSDefault(VertInOut vert_in) return vert_out; } +/* used to prevent internal GPU precision issues width fmod in particular */ +#define PRECISION_OFFSET 0.1 + float4 PSPlanar420(VertInOut vert_in) : TARGET { #ifdef _OPENGL @@ -62,16 +65,17 @@ float4 PSPlanar420(VertInOut vert_in) : TARGET #endif float byte_offset = floor((v_mul + vert_in.uv.x) * width) * 4.0; + byte_offset += PRECISION_OFFSET; float2 sample_pos[4]; if (byte_offset < u_plane_offset) { #ifdef DEBUGGING - return float4(1.0f, 1.0f, 1.0f, 1.0f); + return float4(1.0, 1.0, 1.0, 1.0); #endif - float lum_u = fmod(byte_offset, width) * width_i; - float lum_v = floor(byte_offset * width_i) * height_i; + float lum_u = floor(fmod(byte_offset, width)) * width_i; + float lum_v = floor(byte_offset * width_i) * height_i; /* move to texel centers to sample the 4 pixels properly */ lum_u += width_i * 0.5; @@ -85,16 +89,16 @@ float4 PSPlanar420(VertInOut vert_in) : TARGET } else { #ifdef DEBUGGING return ((byte_offset < v_plane_offset) ? - float4(0.5f, 0.5f, 0.5f, 0.5f) : - float4(0.2f, 0.2f, 0.2f, 0.2f)); + float4(0.5, 0.5, 0.5, 0.5) : + float4(0.2, 0.2, 0.2, 0.2)); #endif float new_offset = byte_offset - ((byte_offset < v_plane_offset) ? u_plane_offset : v_plane_offset); - float ch_u = fmod(new_offset, width_d2) * width_d2_i; - float ch_v = floor(new_offset * width_d2_i) * height_d2_i; + float ch_u = floor(fmod(new_offset, width_d2)) * width_d2_i; + float ch_v = floor(new_offset * width_d2_i) * height_d2_i; float width_i2 = width_i*2.0; /* move to the borders of each set of 4 pixels to force it