Stylized 3D Outline shader

Simply create a ColorRect in a 3D scene, set it to full rect, turn the mouse filter off, create a new shader material and create a new shader then paste the code and enjoy!

Shader code

shader_type canvas_item;

uniform sampler2D SCREEN_TEXTURE : hint_screen_texture, filter_linear_mipmap;

// Style controls
uniform float contrast   = 1.15;
uniform float brightness = 0.02;
uniform float saturation = 1.1;

// Posterization bands
uniform float band1 = 0.25;
uniform float band2 = 0.50;
uniform float band3 = 0.75;

// Halftone controls
uniform float dot_scale    = 120.0;
uniform float dot_strength = 0.55;

// Outline controls
uniform float outline_thickness = 1.0;
uniform float outline_strength  = 1.0;

float luminance(vec3 c) {
    return dot(c, vec3(0.299, 0.587, 0.114));
}

vec3 to_hsv(vec3 c) {
    float maxc = max(c.r, max(c.g, c.b));
    float minc = min(c.r, min(c.g, c.b));
    float d = maxc - minc;

    float h = 0.0;
    if (d > 0.00001) {
        if (maxc == c.r) {
            h = (c.g - c.b) / d;
        } else if (maxc == c.g) {
            h = 2.0 + (c.b - c.r) / d;
        } else {
            h = 4.0 + (c.r - c.g) / d;
        }
        h = h / 6.0;
        if (h < 0.0) h += 1.0;
    }

    float s = maxc == 0.0 ? 0.0 : d / maxc;
    float v = maxc;

    return vec3(h, s, v);
}

vec3 to_rgb(vec3 hsv) {
    float h = hsv.x * 6.0;
    float s = hsv.y;
    float v = hsv.z;

    int i = int(floor(h));
    float f = h - float(i);
    float p = v * (1.0 - s);
    float q = v * (1.0 - s * f);
    float t = v * (1.0 - s * (1.0 - f));

    if (i == 0) return vec3(v, t, p);
    if (i == 1) return vec3(q, v, p);
    if (i == 2) return vec3(p, v, t);
    if (i == 3) return vec3(p, q, v);
    if (i == 4) return vec3(t, p, v);
    return vec3(v, p, q);
}

vec3 adjust_color(vec3 color) {
    color = (color - 0.5) * contrast + 0.5 + brightness;

    vec3 hsv = to_hsv(color);
    hsv.y *= saturation;
    return to_rgb(hsv);
}

float halftone_pattern(vec2 uv, float tone) {
    float angle = 0.5;
    mat2 rot = mat2(
        vec2(cos(angle), -sin(angle)),
        vec2(sin(angle),  cos(angle))
    );
    uv = rot * uv;

    uv *= dot_scale;

    float d = length(fract(uv) - 0.5);
    float radius = mix(0.45, 0.05, tone);
    float mask = smoothstep(radius, radius - 0.02, d);
    return mask;
}

vec3 posterize_color(vec3 col, float l) {
    float band;
    if (l < band1) {
        band = 0.15;
    } else if (l < band2) {
        band = 0.40;
    } else if (l < band3) {
        band = 0.65;
    } else {
        band = 0.9;
    }

    float scale = band / max(l, 0.001);
    return col * scale;
}

float sobel_edge(vec2 uv) {
    vec2 texel = vec2(1.0 / float(textureSize(SCREEN_TEXTURE, 0).x),
                      1.0 / float(textureSize(SCREEN_TEXTURE, 0).y));

    float tl = luminance(texture(SCREEN_TEXTURE, uv + texel * vec2(-1, -1)).rgb);
    float  l = luminance(texture(SCREEN_TEXTURE, uv + texel * vec2(-1,  0)).rgb);
    float bl = luminance(texture(SCREEN_TEXTURE, uv + texel * vec2(-1,  1)).rgb);

    float  t = luminance(texture(SCREEN_TEXTURE, uv + texel * vec2( 0, -1)).rgb);
    float  b = luminance(texture(SCREEN_TEXTURE, uv + texel * vec2( 0,  1)).rgb);

    float tr = luminance(texture(SCREEN_TEXTURE, uv + texel * vec2( 1, -1)).rgb);
    float  r = luminance(texture(SCREEN_TEXTURE, uv + texel * vec2( 1,  0)).rgb);
    float br = luminance(texture(SCREEN_TEXTURE, uv + texel * vec2( 1,  1)).rgb);

    float gx = (tr + 2.0*r + br) - (tl + 2.0*l + bl);
    float gy = (bl + 2.0*b + br) - (tl + 2.0*t + tr);

    return sqrt(gx * gx + gy * gy);
}

void fragment() {
    vec2 uv = SCREEN_UV;
    vec4 scene_col = texture(SCREEN_TEXTURE, uv);

    vec3 col = adjust_color(scene_col.rgb);

    float l = luminance(col);

    vec3 posterized = posterize_color(col, l);

    float tone = 1.0 - l;
    float pattern = halftone_pattern(uv, tone);
    float pattern_mix = smoothstep(0.2, 0.8, tone);

    posterized = mix(posterized, posterized * (1.0 - dot_strength), pattern * pattern_mix);

    float edge = sobel_edge(uv) * outline_strength;

    float outline = smoothstep(0.1, 0.3, edge * outline_thickness);

    vec3 final_color = mix(vec3(0.0), posterized, 1.0 - outline);

    COLOR = vec4(final_color, scene_col.a);
}

Live Preview