Arbitrary Color Reduction and Palette Ordered Dithering

I was looking for a shader like this, but anything similar out there either only supported grayscale, a specified palette, 3bpp/8 color, or had errors cutting off the highest or lowest values. I couldn’t find much info on an actual color recuction algorithm besides the one on the wikipedia article which was pretty vague and I couldn’t get anything near that to work. Anyway I came up with this one, and it works pretty well. Also usable with a specified palette! Unfortunately, GodotShaders doesn’t allow you to paste multiple files, so pay attention to the comments.

To use:

• Add the .gdshaderinc portions to new shader include files.
• Add the .gdshader portion to a new shader file.
• Update the #include sections to match your filepaths.
• Use the ColorRect shader for ColorRects, and the Sprite2D shader for Sprite2Ds.

Caveats:

• Cannot guarantee TextureRect compatibility, since TextureRect UVs are very finnicky.

Updates:

• ColorRect support
• Adjusts to editor zoom scale to reflect in game resolution
• Separated quantization and dither into .gdshaderinc files for better reusability
• Less jitter
• Supports skew, rotation, flipping, centering, and offsets
• Optional handling of subpixels

The uniform parameters are:

• Quantization
  • Quantize Size: Pixel resolution scale (0 is bypass)
  • Handle Scale: Determines how scaled images should display.
  • Handle Subpixels: Auto-scales the quantize size when zoom is < 1, preventing artifacts.
• Dithering
  • Dither Enabled: Turns the dithering on or off.
  • Bayer Pattern: specifies the Bayer matrix pattern to use.
    The higher the number the more detail.
    1 -> 2×2, 2 -> 4×4, 3 -> 8×8
  • Divisions: specifies the number of times each color channel should be split.
    The higher the number the more colors are allowed. (not applicable if a palette is used)
    1 is 3bpp or 8 colors, 2 is 4bpp or 16 colors, etc.
  • Contrast: Modifies the contrast of the image.
  • Shift: Shifts the colors higher or lower.
  • Grayscale: Turns the resulting image to grayscale.
  • Use Palette: Switches between arbitrary color reduction and palette based dithering.
  • Palette: Takes a palette image or you can use a GradientTexture1D and play with the palette gradient in real time! Make sure the width of the Gradient matches the number of colors you’re using!

Feel free to make any suggestions!

Shader code

/*
 * This should be your main shader file if using with a ColorRect.
*/

shader_type canvas_item;
// This file relies on a dither shader include file, listed further down.
// The paths must match your file's location.
#include "res://dither.gdshaderinc"
uniform sampler2D SCREEN_TEXTURE : hint_screen_texture, repeat_disable, filter_nearest;

void vertex() {
    g_q_size = getQuantizeSize(CANVAS_MATRIX);
    g_model_matrix = MODEL_MATRIX;
    g_world_to_clip = SCREEN_MATRIX * CANVAS_MATRIX;
    g_vertex = VERTEX;
}

void fragment() {
    COLOR = ditherScreen(SCREEN_TEXTURE, SCREEN_UV, g_vertex);
}

/*
 * This should be your main shader file if using with a Sprite2D.
*/

shader_type canvas_item;
// This file relies on a dither shader include file, listed further down.
// The paths must match your file's location.
#include "res://dither.gdshaderinc"

void vertex() {
    g_q_size = getQuantizeSize(CANVAS_MATRIX);
    g_model_matrix = MODEL_MATRIX;
    g_texture_size = 1. / TEXTURE_PIXEL_SIZE;
    g_vertex = VERTEX;
    g_flat_vertex = VERTEX;
}

void fragment() {
    COLOR = ditherTexture(TEXTURE, UV);
}

/*
 * This is the dither shader-include file. Please make a copy in a
 * separate file, and update the #include in the main shader to match.
*/

// This file relies on a quantize shader include file, listed further down.
// The paths must match your file's location.
#include "res://quantize.gdshaderinc"
group_uniforms Dithering;
uniform bool dither_enabled = false;
// The Bayer matrix pattern to use
// 1 -> 2x2, 2 -> 4x4, 3 -> 8x8
uniform float bayer_pattern : hint_range(1, 3, 1.0) = 1;
uniform float divisions : hint_range(1, 16, 1.0) = 1.0;
uniform float contrast : hint_range(0, 4) = 1.0;
uniform float shift : hint_range (-1, 1) = 0;
uniform bool grayscale;
uniform bool use_palette;
uniform sampler2D palette;
group_uniforms;

const int matrix2[4] = {
    0, 2,
    3, 1
};
const int matrix4[16] = {
    0,  8,  2,  10,
    12, 4,  14, 6,
    3,  11, 1,  9,
    15, 7,  13, 5
};
const int matrix8[64] = {
    0,  32, 8,  40, 2,  34, 10, 42,
    48, 16, 56, 24, 50, 18, 58, 26,
    12, 44, 4,  36, 14, 46, 6,  38,
    60, 28, 52, 20, 62, 30, 54, 22,
    3,  35, 11, 43, 1,  33, 9,  41,
    51, 19, 59, 27, 49, 17, 57, 25,
    15, 47, 7,  39, 13, 45, 5,  37,
    63, 31, 55, 23, 61, 29, 53, 21
};

int getBayerValue(float dSize, vec2 in_world_vertex) {
    int iDSize = int(dSize);
    float q_size = max(g_q_size, 1.);
    ivec2 iv = ivec2(mod(in_world_vertex, dSize * q_size));
    iv /= int(q_size);
    int index = iv.x + (iv.y * iDSize);

    switch (iDSize) {
        case 2:
            return matrix2[index];
        case 4:
            return matrix4[index];
        default:
            return matrix8[index];
    }
}

vec4 dither_internal(vec4 in_c, vec2 in_world_vertex, bool in_cut_alpha) {
    if (!dither_enabled) {
        return in_c;
    }

    vec4 c = in_c;

    if (in_cut_alpha) {
        c.rgb = vec3(1. - c.a);
    }

    // Get dither size based on matrix selected
    float dSize = pow(2.0, bayer_pattern);
    float dSquared = dSize * dSize;
    // Get space between divisions
    float div = divisions;

    if (use_palette) {
        ivec2 pSize = textureSize(palette, 0);
        pSize /= pSize.y;
        div = float(pSize.x) - 1.0;

        c.rgb = vec3((c.r * 0.299) + (c.g * 0.587) + (c.b * 0.114));
    }

    // add contrast and shift
    c = (c - 0.5 + shift) * contrast + 0.5;
    c = clamp(c, 0.0, 1.0);

    // get dither value
    int d = getBayerValue(dSize, in_world_vertex);

    // normalize the dither value
    float dN = (float(d) / dSquared) - 0.5;
    // get the normalizing value for the division gap
    float r = 1.0 / div;
    float l = r / (dSquared * 2.);

    c += r * dN + l; // add dither value to color
    c *= div; // convert normalized color to quantized range
    c = round(c); // round to nearest available color
    c /= div; // normalize again

    if (use_palette) {
        c.rgb = texture(palette, vec2(c.r, 0.5)).rgb;
    }

    if (grayscale) {
        c.rgb = vec3((c.r + c.g + c.b) / 3.0);
    }

    if (in_cut_alpha) {
        return vec4(vec3(0), 1. - c.r);
    }

    return vec4(c.rgb, c.a);
}

vec4 ditherScreen(sampler2D in_texture, vec2 in_uv, vec2 in_vertex) {
    vec2 world_vertex;
    vec4 c = quantizeScreen(in_texture, in_uv, in_vertex, world_vertex);
    return dither_internal(c, world_vertex, false);
}

vec4 ditherTexture(sampler2D in_texture, vec2 in_uv) {
    vec2 world_vertex;
    vec4 c = quantizeTexture(in_texture, in_uv, world_vertex);
    return dither_internal(c, world_vertex, false);
}

/*
 * This is the quantize shader-include file. Please make a copy in a
 * separate file, and update the #include in the dither shader-include to match.
*/

group_uniforms Quantization;
uniform float quantize_size : hint_range(0,100, 1.0) = 1;
uniform bool handle_scale;
uniform bool handle_subpixels = true;
group_uniforms;

varying mat4 g_model_matrix;
varying mat4 g_world_to_clip;
varying vec2 g_texture_size;
varying vec2 g_vertex;
varying flat vec2 g_flat_vertex;
varying float g_q_size;

float getQuantizeSize(mat4 canvas_matrix) {
    vec2 g_zoom = vec2(length(canvas_matrix[0].xyz), length(canvas_matrix[1].xyz));
    float q_size = quantize_size;

    if (handle_subpixels && g_zoom.x < 1.) {
        q_size = round(quantize_size * (1. / g_zoom.x));
    }

    return q_size;
}

vec4 quantizeTexture(
    sampler2D in_texture,
    vec2 in_uv,
    out vec2 out_world_vertex
) {
    if (g_q_size == 0.) {
        out_world_vertex = (g_model_matrix * vec4(in_uv * g_texture_size, 0, 1)).xy;
        return texture(in_texture, in_uv);
    }

    float q_size = g_q_size;

    // gets -1 if not flipped, 1 if flipped
    vec2 flipped = sign(g_flat_vertex - g_vertex);
    // gets the offset of the texture
    // adds 1 when flipped
    vec2 offset = g_flat_vertex / g_texture_size;

    vec2 uv = in_uv;
    // add offset so positioned back at 0,0 locally
    // so grid lines up
    uv *= flipped;
    uv = offset - uv;
    // scale to texture size
    uv *= g_texture_size;

    if (handle_scale) { // convert to world space
        uv = (g_model_matrix * vec4(uv, 0, 1)).xy;
    }

    out_world_vertex = uv;
    // quantize to specified pixel size
    uv /= q_size;
    uv = floor(uv) + 0.5;
    uv *= q_size;

    if (handle_scale) { // convert back to local space
        uv = (inverse(g_model_matrix) * vec4(uv, 0, 1)).xy;
    }

    // normalize
    uv /= g_texture_size;
    // remove offset so pixel taken at correct location
    uv = offset - uv;
    uv *= flipped;
    return texture(in_texture, uv);
}

vec4 quantizeScreen(
    sampler2D in_screen_texture,
    vec2 in_screen_uv,
    vec2 in_vertex,
    out vec2 out_world_vertex
) {
    out_world_vertex = (g_model_matrix * vec4(in_vertex, 0, 1)).xy;

    if (g_q_size == 0.) {
        return texture(in_screen_texture, in_screen_uv);
    }

    vec2 uv = out_world_vertex;
    uv /= g_q_size;
    uv = floor(uv) + 0.5;
    uv *= g_q_size;
    uv = (g_world_to_clip * vec4(uv, 0, 1)).xy;
    uv = uv * 0.5 + 0.5;
    return texture(in_screen_texture, uv);
}

Tags