I am trying to create a renderer for simple 2D primitives. I followed a tutorial that allowed me to render filled ellipses and rectanges only using one rectangle in OpenGL. It transforms the rectangle and sets the fragment shader to only draw a fragment if it's distance is less than or equal to the radius (the fragment gets passed a position from the non-scaled rectangle).

I've already added rotation, and now I'm trying to draw an outlined ellipse using a similar method: Only draw if it is within the radius and the radius - thickness. (I adjust the bounding box to be larger by half the thickness beforehand).

This method kind of works, but it slices off the outline near the edges.

I can't quite figure out what I need to change to get this to work properly.

Vertex shader:

#version 330 core
layout (location = 0) in vec2 posIn;
uniform vec2 posUL;
uniform vec2 scaleWH;
uniform float rotation;
out vec2 posFrag;

// 2D rotation function
vec2 rotate(vec2 v, float angle) {
    float c = cos(angle);
    float s = sin(angle);
    return vec2(v.x * c - v.y * s, v.x * s + v.y * c);
}

void main()
{
   // Apply Rotation
   vec2 rotatedPos = rotate(posIn, rotation);

   // Apply Scaling and Translation
   gl_Position = vec4(
      rotatedPos.x * scaleWH.x + posUL.x,
      rotatedPos.y * scaleWH.y - posUL.y,
      0.0,
      1.0
   );

   posFrag = posIn + vec2(-1.0f, 1.0f);
}

Fragment shader:

#version 330 core
in vec2 posFrag;
uniform float thickness;
uniform float sizePixel;
uniform vec4 drawColor;
uniform int shapeType;
out vec4 FragColor;

//Functions for drawing each shape
void fillOval() {
   float distance = 1.0 - sqrt(posFrag.x * posFrag.x + posFrag.y * posFrag.y) + sizePixel / 2.0;
   float alpha = clamp(distance / sizePixel, 0.0, 1.0);
   if (alpha < 0.01f)
      discard;
   FragColor  = vec4(drawColor.xyz, drawColor.w * alpha);
}

void fillRectangle() {
   FragColor = drawColor;
}

void outlineOval() {

   // Calculate the thickness in Normalized Device Coordinates 
   float thicknessNDC = thickness * sizePixel;

   float distanceToCenter = length(posFrag);
   float sdf = distanceToCenter - 1.0; 

   float distanceToOutline = abs(sdf) - thicknessNDC * 0.5;
   float alpha = clamp(-distanceToOutline / sizePixel, 0.0, 1.0);

   if (alpha < 0.01f) {
      discard;
   }
    
   FragColor = vec4(drawColor.xyz, drawColor.w * alpha);
}

void outlineRectangle() {
   // todo: implement
   FragColor = drawColor;
}


//Draw the shape, default to rectangle
void main()
{
   switch (shapeType) {
      case 1: fillOval();  break;
      case 2: fillRectangle(); break;
      case 3: outlineOval(); break;
      case 4: outlineRectangle(); break;
      default: fillRectangle();  break;
   }
}

Generic Draw Shape function from renderer:

void Renderer::DrawShape(Vec2 position, Vec2 size, int shapeType, double thickness, double rotation) {
    if (size.x > 0 && size.y > 0) {
        //Get the dimensions of the window.
        iVec2 windowSize = Game::GetInstance().GetWindowSize();

        if (windowSize.x > 0 && windowSize.y > 0) {

            //The width and height of the rectangle in OpenGL's coordinate system.
            Vec2 padding = Vec2(thickness, thickness) * 1.5f;
            size += padding; 
            Vec2 sizeGL = size / windowSize;

            //The upper left position of the rectangle in OpenGL's coordinate system.
            position -= padding * 0.5f;
            Vec2 positionGL = Vec2(
                -1.0f + position.x / windowSize.x * 2.0f,
                -1.0f + position.y / windowSize.y * 2.0f
            );

            float sizePixel = 1.0f / size.x + 1.0f / size.y;

            //Set the shader program's uniforms.
            shader.Use();
            shader.SetFloat("thickness", thickness);
            shader.SetFloat("rotation", rotation);
            shader.SetVec2("posUL", positionGL);
            shader.SetVec2("scaleWH", sizeGL);
            shader.SetFloat("sizePixel", sizePixel);
            shader.SetInt("shapeType", shapeType);

            //Draw the shape.
            glBindVertexArray(VAO);
            glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
            // Unbind vao
            glBindVertexArray(0);
        }
    }
}

Renderer initialization

void Renderer::Init()
{
    Filesystem &fs = Filesystem::GetInstance();

    // Create primitive shaders
    shader.Destroy(); // Destroy just in case it already exists.
    shader = Shader(
        fs.LoadFile("./assets/shaders/renderer/primitivesVert.glsl"),
        fs.LoadFile("./assets/shaders/renderer/primitivesFrag.glsl")
    );

    // Create buffers
    float vertices[] = {
        2.0f,  0.0f,    // Right Top
        2.0f, -2.0f,    // Right Bottom
        -0.0f, -2.0f,   // Left Bottom
        -0.0f,  0.0f    // Left Top
    };

    unsigned int indices[] = {
        0, 1, 3,    //First Triangle
        1, 2, 3     //Second Triangle
    };

    glGenVertexArrays(1, &VAO);
    glGenBuffers(1, &VBO);
    glGenBuffers(1, &EBO);

    glBindVertexArray(VAO);

    glBindBuffer(GL_ARRAY_BUFFER, VBO);
    glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);

    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, EBO);
    glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(indices), indices, GL_STATIC_DRAW);

    glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), (void*)0);
    glEnableVertexAttribArray(0);

    // Unbind
    glBindVertexArray(0);
}

And as for the tutorial I followed, you have to enter your email to see the source, so I'll provide you an excerpt to save you the trouble if you're interested.

const char* RendererGL::vertexShaderSource = "#version 330 core\n"
"layout (location = 0) in vec2 posIn;\n"
"uniform vec2 posUL;\n"
"uniform vec2 scaleWH;\n"
"out vec2 posFrag;\n"

"void main()\n"
"{\n"
"   gl_Position = vec4(posIn.x * scaleWH.x + posUL.x, posIn.y * scaleWH.y - posUL.y, 0.0, 1.0);\n"
"   posFrag = posIn + vec2(-1.0f, 1.0f);\n"
"}\0";


const char* RendererGL::fragmentShaderSource = "#version 330 core\n"
"in vec2 posFrag;\n"
"uniform float sizePixel;\n"
"uniform vec4 drawColor;\n"
"uniform int shapeType;\n"
"out vec4 FragColor;\n"

//Functions for drawing each shape
"void fillOval() {\n"
"    float distance = 1.0 - sqrt(posFrag.x * posFrag.x + posFrag.y * posFrag.y) + sizePixel / 2.0;\n"
"    float alpha = clamp(distance / sizePixel, 0.0, 1.0);\n"
"    if (alpha < 0.01f)\n"
"        discard;\n"
"    FragColor  = vec4(drawColor.xyz, drawColor.w * alpha);\n"
"}\n"

"void fillRectangle() {\n"
"    FragColor = drawColor;\n"
"}\n"


//Draw the shape, default to rectangle
"void main()\n"
"{\n"
"    switch (shapeType) {\n"
"        case 1:  fillOval();  break;\n"
"        default: fillRectangle();  break;\n"
"    }\n"
"}\n\0";

// Shader loading code ...

void RendererGL::loadBuffers() {
    float vertices[] = {
        2.0f,  0.0f,    // Right Top
        2.0f, -2.0f,    // Right Bottom
        0.0f, -2.0f,    // Left Bottom
        0.0f,  0.0f     // Left Top
    };

    unsigned int indices[] = {
        0, 1, 3,    //First Triangle
        1, 2, 3     //Second Triangle
    };

    glGenVertexArrays(1, &VAO);
    glGenBuffers(1, &VBO);
    glGenBuffers(1, &EBO);

    glBindVertexArray(VAO);

    glBindBuffer(GL_ARRAY_BUFFER, VBO);
    glBufferData(GL_ARRAY_BUFFER, sizeof(vertices), vertices, GL_STATIC_DRAW);

    glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, EBO);
    glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(indices), indices, GL_STATIC_DRAW);

    glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), (void*)0);
    glEnableVertexAttribArray(0);
}

// Abstracted drawing functions...

void RendererGL::fillShape(const SDL_Rect * rect, int shapeType) {
    if (rect != nullptr && rect->w > 0 && rect->h > 0 && window != nullptr) {

        //Get the dimensions of the window.
        int windowWidth = 0, windowHeight = 0;
        SDL_GetWindowSize(window, &windowWidth, &windowHeight);
        if (windowWidth > 0 && windowHeight > 0) {

            //The width and height of the rectangle in OpenGL's coordinate system.
            float widthGL = (float)rect->w / windowWidth;
            float heightGL = (float)rect->h / windowHeight;

            //The upper left position of the rectangle in OpenGL's coordinate system.
            float xGL = -1.0f + (float)rect->x / windowWidth * 2.0f;
            float yGL = -1.0f + (float)rect->y / windowHeight * 2.0f;

            //Set the shader program's uniform's.
            glUniform2f(glGetUniformLocation(shaderProgramID, "posUL"), xGL, yGL);
            glUniform2f(glGetUniformLocation(shaderProgramID, "scaleWH"), widthGL, heightGL);
            //The approximate size of a pixel in OpenGL's coordinate system.
            glUniform1f(glGetUniformLocation(shaderProgramID, "sizePixel"),
                1.0f / rect->w + 1.0f / rect->h);
            glUniform1i(glGetUniformLocation(shaderProgramID, "shapeType"), shapeType);

            //Draw the shape.
            glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
        }
    }
}