Post Process Anti Ailiasing
A post processing, edge detection based anti ailiasing solution (dubbed AHAA) similiar to the sub-pixel morphological AA method (SMAA) to reduce aliasing while not blurring visual quality. Not to be used with any anti-ailiasing enabled.
Note: You MUST set the render_priority of the ShaderMaterial to -128, or else you will experience graphical glitches.
Code originally found on Github -> https://github.com/godotengine/godot-proposals/issues/2779
Shader code
shader_type spatial;
render_mode unshaded, depth_prepass_alpha;
void vertex() {
POSITION = vec4(VERTEX, 1.0);
}
uniform sampler2D BackBufferTex : hint_screen_texture, repeat_disable, filter_nearest;
uniform sampler2D DepthBufferTex : hint_depth_texture, repeat_disable, filter_nearest;
uniform sampler2D NormalBufferTex: hint_normal_roughness_texture, repeat_disable, filter_nearest;
uniform float luma_threshold: hint_range(0, 1) = 0.375;
uniform float low_threshold: hint_range(0, 1) = 0.05;
uniform float high_threshold: hint_range(0, 1) = 0.2;
uniform float depth_threshold: hint_range(0, 1) = 0.1;
uniform float near = 0.05; // Set to "near" of camera, doesn't actually do anything
uniform float far = 4000.0; // Set to "far" of camera
uniform float epsilon = 0.001; // Avoid division by zero
uniform int destab_iter = 100;
// Constants for offsets
const ivec2 OffSW = ivec2(-1, 1);
const ivec2 OffSE = ivec2(1, 1);
const ivec2 OffNE = ivec2(1, -1);
const ivec2 OffNW = ivec2(-1, -1);
// Scharr operator kernels
const mat3 scharr_kernel_x = mat3(vec3(-3, 0, 3),
vec3(-10, 0, 10),
vec3(-3, 0, 3));
const mat3 scharr_kernel_y = mat3(vec3(-3, -10, -3),
vec3(0, 0, 0),
vec3(3, 10, 3));
const mat3 gaussian_kernel = mat3(vec3(0.0625, 0.125, 0.0625),
vec3(0.125, 0.25, 0.125),
vec3(0.0625, 0.125, 0.0625));
// Function to calculate luminance
float getLuma(vec3 color) {
return dot(color, vec3(0.299, 0.587, 0.114));
}
// Function to calculate chroma
vec3 getChroma(vec3 color) {
float maxComponent = max(max(color.r, color.g), color.b);
return color / (maxComponent + epsilon); // Add epsilon to avoid division by zero
}
// Function to sample luminance at an offset
float sampleLumaOff(vec2 uv, ivec2 offset, vec2 texSize) {
return getLuma(texture(BackBufferTex, uv + vec2(offset) / texSize).rgb);
}
// Sampling functions
vec3 sampleColor(vec2 p) {
return texture(BackBufferTex, p).rgb;
}
vec3 sampleColorFromNormal(vec2 p) {
return texture(NormalBufferTex, p).rgb;
}
// Function to linearize depth value
float linearizeDepth(float depth, vec2 uv, mat4 inv_projection_matrix) {
vec3 ndc = vec3(uv * 2.0 - 1.0, depth);
vec4 view = inv_projection_matrix * vec4(ndc, 1.0);
view.xyz /= view.w;
return -view.z;
}
// Function to calculate depth difference
float getDepthDifference(vec2 uv, vec2 texSize, mat4 inv_projection_matrix, out bool outOfBounds) {
outOfBounds = false;
float centerDepth = linearizeDepth(texture(DepthBufferTex, uv).x, uv, inv_projection_matrix);
if (centerDepth > far) {
outOfBounds = true;
}
float maxDepthDifference = 0.0;
for (int x = -1; x <= 1; x++) {
for (int y = -1; y <= 1; y++) {
if (x == 0 && y == 0) continue; // Skip the center pixel
vec2 offset = vec2(float(x), float(y)) / texSize;
float neighborDepth = linearizeDepth(texture(DepthBufferTex, uv + offset).x, uv + offset, inv_projection_matrix);
if (neighborDepth < far) {
outOfBounds = false;
}
maxDepthDifference = max(maxDepthDifference, abs(centerDepth - neighborDepth));
}
}
return maxDepthDifference / (centerDepth + epsilon); // Normalize depth difference
}
vec3 gaussianBlur(vec2 uv, vec2 texSize) {
vec3 colorSum = vec3(0.0);
for (int x = -1; x <= 1; x++) {
for (int y = -1; y <= 1; y++) {
vec2 offset = vec2(float(x), float(y)) / texSize;
colorSum += texture(BackBufferTex, uv + offset).rgb * gaussian_kernel[x + 1][y + 1];
}
}
return colorSum;
}
// Function to apply Scharr filter
vec2 applyScharr(vec2 uv, vec2 texSize) {
float gx = 0.0;
float gy = 0.0;
for (int x = -1; x <= 1; x++) {
for (int y = -1; y <= 1; y++) {
vec3 sampleColor = sampleColor(uv + vec2(float(x), float(y)) / texSize);
float luma = getLuma(sampleColor);
gx += luma * scharr_kernel_x[x + 1][y + 1];
gy += luma * scharr_kernel_y[x + 1][y + 1];
}
}
return vec2(gx, gy);
}
// Function to apply Scharr filter
vec2 applyScharrToNormal(vec2 uv, vec2 texSize) {
float gx = 0.0;
float gy = 0.0;
for (int x = -1; x <= 1; x++) {
for (int y = -1; y <= 1; y++) {
vec3 sampleColor = sampleColorFromNormal(uv + vec2(float(x), float(y)) / texSize);
float luma = getLuma(sampleColor);
gx += luma * scharr_kernel_x[x + 1][y + 1];
gy += luma * scharr_kernel_y[x + 1][y + 1];
}
}
return vec2(gx, gy);
}
const mat3 prewitt_kernel_x = mat3(
vec3(-1, 0, 1),
vec3(-1, 0, 1),
vec3(-1, 0, 1)
);
const mat3 prewitt_kernel_y = mat3(
vec3(1, 1, 1),
vec3(0, 0, 0),
vec3(-1, -1, -1)
);
vec2 applyPrewitt(vec2 uv, vec2 tex_size) {
vec2 kernel_size = vec2(3, 3);
vec2 half_kernel = kernel_size / 2.0;
float sum_x = 0.0;
float sum_y = 0.0;
for (int i = 0; i < 3; i++) {
for (int j = 0; j < 3; j++) {
vec2 uv_offset = (vec2(float(i), float(j)) - half_kernel) / tex_size;
vec4 texel = texture(BackBufferTex, uv + uv_offset);
float intensity = (texel.r + texel.g + texel.b) / 3.0;
sum_x += prewitt_kernel_x[i][j] * intensity;
sum_y += prewitt_kernel_y[i][j] * intensity;
}
}
return vec2(sum_x, sum_y);
}
vec2 applyPrewittToNormal(vec2 uv, vec2 tex_size) {
vec2 kernel_size = vec2(3, 3);
vec2 half_kernel = kernel_size / 2.0;
float sum_x = 0.0;
float sum_y = 0.0;
for (int i = 0; i < 3; i++) {
for (int j = 0; j < 3; j++) {
vec2 uv_offset = (vec2(float(i), float(j)) - half_kernel) / tex_size;
vec4 texel = texture(NormalBufferTex, uv + uv_offset);
float intensity = (texel.r + texel.g + texel.b) / 3.0;
sum_x += prewitt_kernel_x[i][j] * intensity;
sum_y += prewitt_kernel_y[i][j] * intensity;
}
}
return vec2(sum_x, sum_y);
}
void fragment() {
vec2 texSize = vec2(textureSize(BackBufferTex, 0));
vec2 uv = SCREEN_UV;
vec2 RCP2 = 2.0 / texSize;
// Depth difference for depth-aware blending
bool outOfBounds;
float depthDifference = getDepthDifference(uv, texSize, INV_PROJECTION_MATRIX, outOfBounds);
if (outOfBounds) {
discard;
}
bool depthEdge = depthDifference > depth_threshold;
// Scharr-based edge detection
vec2 gradient = applyScharr(uv, texSize);
float gradientMagnitude = length(gradient);
vec2 normalGradient = applyScharrToNormal(uv, texSize);
float normalGradientMagnitude = length(normalGradient);
//vec2 prewittGradient = applyPrewitt(uv, texSize);
//float prewittGradientMagnitude = length(gradient);
//vec2 prewittNormalGradient = applyPrewittToNormal(uv, texSize);
//float prewittNormalGradientMagnitude = length(normalGradient);
bool isStrongEdge = gradientMagnitude >= high_threshold;// || prewittGradientMagnitude >= high_threshold;
bool isWeakEdge = (gradientMagnitude >= low_threshold && gradientMagnitude < high_threshold);// || (prewittGradientMagnitude >= low_threshold && prewittGradientMagnitude < high_threshold);
bool isNormalStrongEdge = normalGradientMagnitude >= high_threshold;// || prewittNormalGradientMagnitude >= high_threshold;
bool isNormalWeakEdge = (normalGradientMagnitude >= low_threshold && normalGradientMagnitude < high_threshold);// || (prewittNormalGradientMagnitude >= low_threshold && prewittNormalGradientMagnitude < high_threshold);
bool isConnectedToStrongEdge = false;
if (isWeakEdge || isNormalWeakEdge) {
for (int x = -1; x <= 1; x++) {
for (int y = -1; y <= 1; y++) {
if (x == 0 && y == 0) continue; // Skip the center pixel
vec2 neighborUV = uv + vec2(float(x), float(y)) / texSize;
vec2 neighborGradient = applyScharr(neighborUV, texSize); //neighborUV instead of uv here made it worse?
float neighborStrength = length(neighborGradient);
vec2 neighborNormalGradient = applyScharrToNormal(neighborUV, texSize); //neighborUV instead of uv here made it worse?
float neighborNormalStrength = length(neighborNormalGradient);
if (neighborStrength > high_threshold || neighborNormalStrength > high_threshold) {
isConnectedToStrongEdge = true;
break;
}
}
if (isConnectedToStrongEdge) break;
}
}
bool cannyEdge = isStrongEdge || isNormalStrongEdge || (isWeakEdge && isConnectedToStrongEdge) || (isNormalWeakEdge && isConnectedToStrongEdge);
// Additional luminance-based edge refinement
vec4 lumaA;
lumaA.x = sampleLumaOff(uv, OffSW, texSize);
lumaA.y = sampleLumaOff(uv, OffSE, texSize);
lumaA.z = sampleLumaOff(uv, OffNE, texSize);
lumaA.w = sampleLumaOff(uv, OffNW, texSize);
float gradientSWNE = lumaA.x - lumaA.z;
float gradientSENW = lumaA.y - lumaA.w;
vec2 dir = vec2(gradientSWNE + gradientSENW, gradientSWNE - gradientSENW);
vec2 dirM = abs(dir);
float dirMMin = min(dirM.x, dirM.y);
vec2 offM = clamp(vec2(0.0625) * dirM / dirMMin, 0.0, 1.0);
vec2 offMult = RCP2 * sign(dir);
bool passC;
float offMMax = max(offM.x, offM.y);
vec4 lumaAC = lumaA;
if (abs(offMMax - 1.0) < 0.0001) {
bool horSpan = abs(offM.x - 1.0) < 0.0001;
bool negSpan = horSpan ? offMult.x < 0.0 : offMult.y < 0.0;
bool sowSpan = horSpan == negSpan;
vec2 uvC = uv;
if (horSpan) uvC.x += 2.0 * offMult.x;
if (!horSpan) uvC.y += 2.0 * offMult.y;
if (sowSpan) lumaAC.x = sampleLumaOff(uvC, OffSW, texSize);
if (!negSpan) lumaAC.y = sampleLumaOff(uvC, OffSE, texSize);
if (!sowSpan) lumaAC.z = sampleLumaOff(uvC, OffNE, texSize);
if (negSpan) lumaAC.w = sampleLumaOff(uvC, OffNW, texSize);
float gradientSWNEC = lumaAC.x - lumaAC.z;
float gradientSENWC = lumaAC.y - lumaAC.w;
vec2 dirC = vec2(gradientSWNEC + gradientSENWC, gradientSWNEC - gradientSENWC);
if (!horSpan) dirC = dirC.yx;
passC = abs(dirC.x) > 2.0 * abs(dirC.y);
if (passC) offMult *= 2.0;
}
// Combine edge detections: Depth-based, Canny-based, Additional Luminance-based
int edge = 0;
if (depthEdge) edge += 1;
if (cannyEdge) edge += 1;
//if (abs(offMMax - 1.0) < 0.0001 && passC) edge += 1;
bool isEdge = (edge > 0);
// Blend colors based on combined edge detection
vec3 finalColor;
float finalAlpha = 1.0;
if (isEdge) {
// Collect neighborhood colors and chroma
vec3 neighborhoodColors[9];
float neighborhoodLuma[9];
vec3 neighborhoodChroma[9];
int index = 0;
for (int x = -1; x <= 1; x++) {
for (int y = -1; y <= 1; y++) {
vec2 offset = vec2(float(x), float(y)) / texSize;
vec3 color = texture(BackBufferTex, uv + offset).rgb;
neighborhoodColors[index] = color;
neighborhoodLuma[index] = getLuma(color);
neighborhoodChroma[index] = getChroma(color);
index++;
}
}
// Calculate local variance
vec3 rgbM = sampleColor(uv);
float localVariance = 0.0;
for (int i = 0; i < 9; i++) {
localVariance += distance(rgbM, neighborhoodColors[i]);
}
localVariance /= 9.0;
// Calculate dynamic threshold based on local variance
float dynamicThreshold = mix(luma_threshold, 1.0, localVariance); // Mix based on local variance
if (localVariance > dynamicThreshold) {
discard;
}
// Advanced blending logic
vec2 offset = offM * offMult;
vec3 rgbN = sampleColor(uv - offset);
vec3 rgbP = sampleColor(uv + offset);
// Chroma check
float lumaMin = min(min(min(min(min(min(min(min(neighborhoodLuma[0], neighborhoodLuma[1]), neighborhoodLuma[2]), neighborhoodLuma[3]), neighborhoodLuma[4]), neighborhoodLuma[5]), neighborhoodLuma[6]), neighborhoodLuma[7]), neighborhoodLuma[8]);
float lumaACMin = min(min(lumaAC.x, lumaAC.y), min(lumaAC.z, lumaAC.w));
float lumaMax = max(max(max(max(max(max(max(max(neighborhoodLuma[0], neighborhoodLuma[1]), neighborhoodLuma[2]), neighborhoodLuma[3]), neighborhoodLuma[4]), neighborhoodLuma[5]), neighborhoodLuma[6]), neighborhoodLuma[7]), neighborhoodLuma[8]);
float lumaACMax = max(max(lumaAC.x, lumaAC.y), max(lumaAC.z, lumaAC.w));
lumaMin = min(lumaMin, lumaACMin);
lumaMax = max(lumaMax, lumaACMax);
vec3 chromaMin = min(min(min(min(min(min(min(min(neighborhoodChroma[0], neighborhoodChroma[1]), neighborhoodChroma[2]), neighborhoodChroma[3]), neighborhoodChroma[4]), neighborhoodChroma[5]), neighborhoodChroma[6]), neighborhoodChroma[7]), neighborhoodChroma[8]);
vec3 chromaMax = max(max(max(max(max(max(max(max(neighborhoodChroma[0], neighborhoodChroma[1]), neighborhoodChroma[2]), neighborhoodChroma[3]), neighborhoodChroma[4]), neighborhoodChroma[5]), neighborhoodChroma[6]), neighborhoodChroma[7]), neighborhoodChroma[8]);
bool withinRange = false;
for (int i = 0; i < destab_iter; i++) {
float mixmul = clamp(0.4 + ((float(i) * 0.6 / float(destab_iter)) * float(((i % 2) * 2) - 1)), 0.0, 1.0);
vec3 rgbR = (rgbN + rgbP) * (1.0 - mixmul)/2.0 + rgbM * mixmul;
float lumaR = getLuma(rgbR);
vec3 chromaR = getChroma(rgbR);
bool lumaOutOfRange = lumaR < lumaMin || lumaR > lumaMax;
bool chromaOutOfRange = any(lessThan(chromaR, chromaMin)) || any(greaterThan(chromaR, chromaMax));
if (!lumaOutOfRange && !chromaOutOfRange) {
finalColor = rgbR;
withinRange = true;
break;
}
}
if (!withinRange) {
discard;
}
} else {
discard; // Use the original color if not an edge
}
ALBEDO = finalColor;
ALPHA = finalAlpha;
}