62ndidiot
Probably you will need to change the Sobel (calc_edge_overlay) edge detection
I fixed Sobel convolution:
for (x = x_min, xdiv3 = x_min/2; x < x_max; x += 4, xdiv3 += 2)
{ // 1 231 2 3
// convolve vert (second Y) // 1 234 5 678 >> 1 2 3 4 5 6
conv1 = *(ptrh1 + x + 1) * ( 1) + // UYVYYY UYVYYY >> UYVY UYVY UYVY
*(ptrh1 + x + 5) * (-1) + // 012345 678901 >> 0123 4567 8901
*(ptrh2 + x + 1) * ( 2) +
*(ptrh2 + x + 5) * (-2) +
*(ptrh3 + x + 1) * ( 1) +
*(ptrh3 + x + 5) * (-1);
if (conv1 < 0) // abs()
conv1 = -conv1;
// convolve vert (first Y of next pixel)
conv2 = *(ptrh1 + x + 1) * ( 1) +
*(ptrh1 + x + 3) * ( 2) +
*(ptrh1 + x + 5) * ( 1) +
*(ptrh3 + x + 1) * (-1) +
*(ptrh3 + x + 3) * (-2) +
*(ptrh3 + x + 5) * (-1);
if (conv2 < 0) // abs()
conv2 = -conv2;
if (conv1 + conv2 > conf.edge_overlay_thresh)
{
bv_set(edgebuf, (y-camera_screen.edge_hmargin)*viewport_width + xdiv3, 1);
}
// Do it once again for the next 'pixel'
// convolve vert (second Y)
conv1 = *(ptrh1 + x + 7) * ( 1) +
*(ptrh1 + x + 11) * (-1) +
*(ptrh2 + x + 7) * ( 2) +
*(ptrh2 + x + 11) * (-2) +
*(ptrh3 + x + 7) * ( 1) +
*(ptrh3 + x + 11) * (-1);
if (conv1 < 0) // abs()
conv1 = -conv1;
// convolve vert (first Y of next pixel)
conv2 = *(ptrh1 + x + 7) * ( 1) +
*(ptrh1 + x + 9) * ( 2) +
*(ptrh1 + x + 11) * ( 1) +
*(ptrh3 + x + 7) * (-1) +
*(ptrh3 + x + 9) * (-2) +
*(ptrh3 + x + 11) * (-1);
if (conv2 < 0) // abs()
conv2 = -conv2;
if (conv1 + conv2 > conf.edge_overlay_thresh)
{
bv_set(edgebuf, (y-camera_screen.edge_hmargin)*viewport_width + xdiv3+1, 1);
}
} // for x
} // for y
Now it looks better:
But I'm not sure about
x step: I was thinking it should be 8, but practically I found it as 4.
