opensource/obs-studio
3
0
Fork 0
mirror of https://github.com/obsproject/obs-studio.git synced 2026-09-24 23:31:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix precision issues with new conversion shader

Browse Source 
Turns out that on some adapters, due to some sort of internal GPU
precision error, fmod(x, y) can return x when x == y, wich is incorrect
(and no, they were actually equal, not off due to precision errors).

This would cause the shader to sample wrong coordinates on the edges
sometimes.  Just adding 0.1 to the x value before being put in to fmod
and then flooring the result after fixes the issue.
This commit is contained in:
jp9000
2014-02-17 09:28:27 -07:00
parent 105aafbc96
commit a6dbe1501d
1 changed files with 11 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/data/libobs/format_conversion.effect
+11 -7
View File
@@ -53,6 +53,9 @@ VertInOut VSDefault(VertInOut vert_in)
return vert_out;
}
/* used to prevent internal GPU precision issues width fmod in particular */
#define PRECISION_OFFSET 0.1
float4 PSPlanar420(VertInOut vert_in) : TARGET
{
#ifdef _OPENGL
@@ -62,16 +65,17 @@ float4 PSPlanar420(VertInOut vert_in) : TARGET
#endif
float byte_offset = floor((v_mul + vert_in.uv.x) * width) * 4.0;
byte_offset += PRECISION_OFFSET;
float2 sample_pos[4];
if (byte_offset < u_plane_offset) {
#ifdef DEBUGGING
return float4(1.0f, 1.0f, 1.0f, 1.0f);
return float4(1.0, 1.0, 1.0, 1.0);
#endif
float lum_u = fmod(byte_offset, width) * width_i;
float lum_v = floor(byte_offset * width_i) * height_i;
float lum_u = floor(fmod(byte_offset, width)) * width_i;
float lum_v = floor(byte_offset * width_i) * height_i;
/* move to texel centers to sample the 4 pixels properly */
lum_u += width_i * 0.5;
@@ -85,16 +89,16 @@ float4 PSPlanar420(VertInOut vert_in) : TARGET
} else {
#ifdef DEBUGGING
return ((byte_offset < v_plane_offset) ?
float4(0.5f, 0.5f, 0.5f, 0.5f) :
float4(0.2f, 0.2f, 0.2f, 0.2f));
float4(0.5, 0.5, 0.5, 0.5) :
float4(0.2, 0.2, 0.2, 0.2));
#endif
float new_offset = byte_offset -
((byte_offset < v_plane_offset) ?
u_plane_offset : v_plane_offset);
float ch_u = fmod(new_offset, width_d2) * width_d2_i;
float ch_v = floor(new_offset * width_d2_i) * height_d2_i;
float ch_u = floor(fmod(new_offset, width_d2)) * width_d2_i;
float ch_v = floor(new_offset * width_d2_i) * height_d2_i;
float width_i2 = width_i*2.0;
/* move to the borders of each set of 4 pixels to force it