Arbitrary Color Reduction and Palette Ordered Dithering

I was looking for a shader like this, but anything similar out there either only supported grayscale, a specified palette, 3bpp/8 color, or had errors cutting off the highest or lowest values. I couldn’t find much info on an actual color recuction algorithm besides the one on the wikipedia article which was pretty vague and I couldn’t get anything near that to work. Anyway I came up with this one, and it works pretty well. Also usable with a specified palette!

To use:

Add the .gdshaderinc portions to new shader include files.
Add the .gdshader portion to a new shader file.
Throw the shader on a ColorRect and you’re good to go!

Updates:

ColorRect support
Adjusts to editor zoom scale to reflect in game resolution
Separated quantization and dither into .gdshaderinc files for better reusability
Less jitter

The uniform parameters are:

Quantize Size: Pixel resolution scale (0 is bypass)
Dither Enabled: Turns the dithering on or off.
Bayer Pattern: specifies the Bayer matrix pattern to use.
The higher the number the more detail.
1 -> 2×2, 2 -> 4×4, 3 -> 8×8
Divisions: specifies the number of times each color channel should be split.
The higher the number the more colors are allowed. (not applicable if a palette is used)
1 is 3bpp or 8 colors, 2 is 4bpp or 16 colors, etc.
Contrast: Modifies the contrast of the image.
Shift: Shifts the colors higher or lower.
Grayscale: Turns the resulting image to grayscale.
Use Palette: Switches between arbitrary color reduction and palette based dithering.
Palette: Takes a palette image or you can use a GradientTexture1D and play with the palette gradient in real time! Make sure the width of the Gradient matches the number of colors you’re using!

Feel free to make any suggestions!

Shader code

// ditherShader.gdshader

shader_type canvas_item;
#include "res://dither.gdshaderinc"
uniform sampler2D SCREEN_TEXTURE : hint_screen_texture, filter_nearest;

void vertex() {
    texture_size = getScreenSize(CANVAS_MATRIX, SCREEN_MATRIX);
    world_vertex = (MODEL_MATRIX * vec4(VERTEX, 0, 1)).xy;
}

void fragment() {
    COLOR = dither(SCREEN_TEXTURE, SCREEN_UV, SCREEN_PIXEL_SIZE, false);
}

// dither.gdshaderinc

#include "res://quantize.gdshaderinc"
uniform bool dither_enabled = false;
// The Bayer matrix pattern to use
// 1 -> 2x2, 2 -> 4x4, 3 -> 8x8
uniform float bayer_pattern : hint_range(1, 3, 1.0) = 1;
uniform float divisions : hint_range(1, 16, 1.0) = 1.0;
uniform float contrast : hint_range(0, 4) = 1.0;
uniform float shift : hint_range (-1, 1) = 0;
uniform bool grayscale;
uniform bool use_palette;
uniform sampler2D palette;

const int matrix2[4] = {
    0, 2,
    3, 1
};
const int matrix4[16] = {
    0,  8,  2,  10,
    12, 4,  14, 6,
    3,  11, 1,  9,
    15, 7,  13, 5
};
const int matrix8[64] = {
    0,  32, 8,  40, 2,  34, 10, 42,
    48, 16, 56, 24, 50, 18, 58, 26,
    12, 44, 4,  36, 14, 46, 6,  38,
    60, 28, 52, 20, 62, 30, 54, 22,
    3,  35, 11, 43, 1,  33, 9,  41,
    51, 19, 59, 27, 49, 17, 57, 25,
    15, 47, 7,  39, 13, 45, 5,  37,
    63, 31, 55, 23, 61, 29, 53, 21
};

int getBayerValue(float dSize) {
    int iDSize = int(dSize);
    float qSize = quantize_size == 0. ? 1. : quantize_size;
    ivec2 iv = ivec2(mod(world_vertex, dSize * qSize));
    iv /= int(qSize);
    int index = iv.x + (iv.y * iDSize);

    switch (iDSize) {
        case 2:
            return matrix2[index];
        case 4:
            return matrix4[index];
        default:
            return matrix8[index];
    }
}

vec4 dither(sampler2D in_texture, vec2 in_uv, vec2 screen_pixel_size, bool cut_alpha) {
    vec2 uv = quantizeUV(in_uv, screen_pixel_size);
    vec4 out_c = textureLod(in_texture, uv, 0);

    if (!dither_enabled) {
        return out_c;
    }

    vec3 c = out_c.rgb;

    if (cut_alpha) {
        c = vec3(1. - out_c.a);
    }

    // Get dither size based on matrix selected
    float dSize = pow(2.0, bayer_pattern);
    float dSquared = dSize * dSize;
    // Get space between divisions
    float div = divisions;

    if (use_palette) {
        ivec2 pSize = textureSize(palette, 0);
        pSize /= pSize.y;
        div = float(pSize.x) - 1.0;

        c = vec3((c.r * 0.299) + (c.g * 0.587) + (c.b * 0.114));
    }

    // add contrast and shift
    c = (c - 0.5 + shift) * contrast + 0.5;
    c = clamp(c, 0.0, 1.0);

    // get dither value
    int d = getBayerValue(dSize);

    // normalize the dither value
    float dN = (float(d) / dSquared) - 0.5;
    // get the normalizing value for the division gap
    float r = 1.0 / div;
    float l = r / (dSquared * 2.);

    c += r * dN + l; // add dither value to color
    c *= div; // convert normalized color to quantized range
    c = round(c); // round to nearest available color
    c /= div; // normalize again

    c = mix(c, texture(palette, vec2(c.r, 0.5)).rgb, float(use_palette));
    c = mix(c, vec3((c.r + c.g + c.b) / 3.0), float(grayscale));

    out_c.rgb = c;
    return out_c;
}

// quantize.gdshaderinc

uniform float quantize_size : hint_range(0,100, 1.0) = 1;
varying vec2 world_vertex;
varying vec2 texture_size;

vec2 getScreenSize(mat4 canvas_matrix, mat4 screen_matrix) {
    vec2 screen_size = 1. / ((screen_matrix * vec4(1, 1, 0, 0)).xy * 0.5);
    vec2 zoom = (canvas_matrix * vec4(1, 1, 0, 0)).xy;
    return screen_size / zoom;
}

vec2 getTextureSize(mat4 model_matrix, vec2 texture_pixel_size) {
    vec2 transform_scale = (model_matrix * vec4(1, 1, 0, 0)).xy;
    return (1. / texture_pixel_size) * transform_scale;
}

vec2 quantizeUV(vec2 in_uv, vec2 screen_pixel_size) {
    if (quantize_size == 0.) {
        return in_uv;
    }

    vec2 wv = world_vertex / quantize_size;
    vec2 v_offset = fract(wv) - 0.5;
    vec2 out_uv = in_uv * texture_size;
    out_uv -= v_offset * quantize_size;
    out_uv /= texture_size;
    return out_uv + (screen_pixel_size * 0.5);
}