Sobel Edge Outline shader per-object

Hey, I believe this is because of how the shader is testing for edges.

If we (mentally) zoom into your image around the edge that isn’t highlighted, what the shader is doing is checking that region of pixels and seeing that their normals are the same. (Both the normal on the cube and on the floor are pointing upwards). This means that the shader doesn’t detect this as an edge, and doesn’t draw an outline there.

A fix for this is to also test against depth changes, using hint_depth_texture. This way, if adjacent pixels are at a greater depth / behind the current pixel, an outline will also be drawn. Below is my implementation.

All this code is CC0; use it however you like.

I would also recommend adding in some antialiasing. Either just turning it on in the godot settings (which is the easiest method), or adding some manual anti aliasing to the code below (e.g. by smoothstepping the edge_total value between 0 and 1 and using that to modify the outline_color).

shader_type spatial;

// references
// https://godotshaders.com/shader/sobel-edge-outline-shader-per-object/

render_mode unshaded;

uniform sampler2D SCREEN_TEXTURE: hint_screen_texture, filter_linear_mipmap_anisotropic;
uniform sampler2D NORMAL_TEXTURE: hint_normal_roughness_texture, filter_linear_mipmap_anisotropic;
uniform sampler2D DEPTH_TEXTURE: hint_depth_texture, filter_linear_mipmap_anisotropic;

uniform float outline_threshold = .002;
/* This also gets divided by 500. later on (i.e. really the default value for this threshold is .002 / 500.). I found this worked better. */
uniform float depth_outline_threshold = .002;
uniform float depth_outline_further_reduction = 500.;
uniform vec3 outline_color : source_color;
uniform vec3 non_outline_color : source_color;

const mat3 sobel_x = mat3(
  vec3(1, 2, 1), // column 1
  vec3(0, 0, 0), // column 2
  vec3(-1, -2, -1) // column 3
);

const mat3 sobel_y = mat3(
  vec3(1, 0, -1), // column 1
  vec3(2, 0, -2), // column 2
  vec3(1, 0, -1) // column 3
);

bool is_edge(in vec2 screen_uv, in vec3 normal, in vec2 screen_pixel_size)
{
  // column index is y, row index is x
  // column 0 is negative y, column 1 is 0 y and column 2 is positive y
  // row 0 is negative x, row 1 is 0 x and row 2 is positive x
  mat3 adjacent_pixels = mat3(0.);
   
  for (int dx = -1; dx <= 1; dx++)
  {
    for (int dy = -1; dy <= 1; dy++)
    {
      vec3 adjacent_normal = texture(NORMAL_TEXTURE, screen_uv + vec2(ivec2(dx, dy)) * screen_pixel_size).rgb;
       
      // for some INSANE reason mat3 is [column][row]. actually ig that makes sense as the x coordinate increases along columns, so the first index “should” index columns
      adjacent_pixels[dy+1][dx+1] = length(adjacent_normal – normal);
    }
  }
   
  float edge_x = 0.;
  float edge_y = 0.;
   
  for (int i = 0; i <= 2; i++)
  {
    edge_x += dot(sobel_x[i], adjacent_pixels[i]);
    edge_y += dot(sobel_y[i], adjacent_pixels[i]);
  }
   
  float edge_total = sqrt(pow(edge_x, 2.) + pow(edge_y, 2.));
   
  return edge_total > outline_threshold;
}

bool is_depth_edge(in vec2 screen_uv, in float depth, in vec2 screen_pixel_size)
{
  mat3 adjacent_pixels = mat3(0.);
   
  for (int dx = -1; dx <= 1; dx++)
  {
    for (int dy = -1; dy <= 1; dy++)
    {
      float adjacent_depth = texture(DEPTH_TEXTURE, screen_uv + vec2(ivec2(dx, dy)) * screen_pixel_size).x;
       
      adjacent_pixels[dy+1][dx+1] = abs(adjacent_depth – depth);
    }
  }
   
  float edge_x = 0.;
  float edge_y = 0.;
   
  for (int i = 0; i <= 2; i++)
  {
    edge_x += dot(sobel_x[i], adjacent_pixels[i]);
    edge_y += dot(sobel_y[i], adjacent_pixels[i]);
  }
   
  float edge_total = sqrt(pow(edge_x, 2.) + pow(edge_y, 2.));
   
  return edge_total > depth_outline_threshold / depth_outline_further_reduction;
}

void fragment()
{
  //vec3 screen_normal = texture(NORMAL_TEXTURE, SCREEN_UV).rgb;
  //screen_normal = screen_normal * 2. – 1.;
   
  vec2 screen_pixel_size = 1./VIEWPORT_SIZE;
   
  bool draw_edge = false;
   
  // test for changes in adjacent pixel normals
  if (is_edge(SCREEN_UV, NORMAL, screen_pixel_size))
  {
    draw_edge = true;
  }
   
  float depth = texture(DEPTH_TEXTURE, SCREEN_UV).x;
   
  // test for changes in depth
  if (is_depth_edge(SCREEN_UV, depth, screen_pixel_size))
  {
    draw_edge = true;
  }
   
  if (draw_edge)
  {
    ALBEDO = outline_color;
  }
  else
  {
    ALBEDO = non_outline_color;
  }
}