Tags

, , , , , , , , , , , , ,

In my last post, OpenGL Shaders using Python, I explained how OpenCV computer vision could inspect stereo images (from two webcams) to figure out which objects are closest to us. OpenGL graphics library could then render 3D shapes on top of the webcam images, using the coordinates of those closest objects to bring them alive in front of our eyes!

The Python OpenGL code from the last post did the following during initialisation:

  1. Set up and compile vertex shader and fragment shader programs for rendering a webcam image
  2. Buffer vertex and UV data for our webcam image
  3. Load our webcam image and create a texture with it

The code then did the following during each draw:

  1. Create and send a projection and modelview matrix to our vertex shader
  2. Bind and send our buffers to our vertex shader
  3. Bind and send our texture to our fragment shader
  4. Draw our webcam image

And here’s our Python OpenGL application rendering our webcam image:

OpenGLShaders_StereoDepth_BackgroundImage

Note the application is using PyOpenGL, a cross-platform Python binding to OpenGL and related APIs.

The code is fairly simple. But next up, we want to draw 3D shapes for the closest objects in our webcam images. That means more buffer objects, alongside the information to render them.

A Vertex Array Object (VAO) can contain one or more buffers and store their rendering information. By using Vertex Array Objects, each draw becomes a breeze, cutting down on the calls we need to make. So let’s amend the code!

Here’s the Vertex Array Object being set up for our background webcam image:

# create and bind VAO
self.backgroundVAO = glGenVertexArrays(1)
glBindVertexArray(self.backgroundVAO)

# generate vertex buffer
vertexBuffer = glGenBuffers(1)
glBindBuffer(GL_ARRAY_BUFFER, vertexBuffer)

# enable vertex attribute array
glEnableVertexAttribArray(aVert)

# vertex attribute pointer
glVertexAttribPointer(aVert, 3, GL_FLOAT, GL_FALSE, 0, None)

# buffer vertex data
vertexData = numpy.array(backgroundVertex, numpy.float32)
glBufferData(GL_ARRAY_BUFFER, 4 * len(vertexData), vertexData, GL_STATIC_DRAW)

# generate UV buffer
uvBuffer = glGenBuffers(1)
glBindBuffer(GL_ARRAY_BUFFER, uvBuffer)

# enable UV attribute array
glEnableVertexAttribArray(aUV)

# UV attribute pointer
glVertexAttribPointer(aUV, 2, GL_FLOAT, GL_FALSE, 0, None)

# buffer UV data
uvData = numpy.array(backgroundUV, numpy.float32)
glBufferData(GL_ARRAY_BUFFER, 4 * len(uvData), uvData, GL_STATIC_DRAW)

We create and bind our Vertex Array Object, before adding the vertex and UV buffers (along with the vertex shader inputs they will use).

Now when we come to draw our background webcam image, it’s as simple as binding the Vertex Array Object and texture:

# bind VAO
glBindVertexArray(self.backgroundVAO)

# bind background texture
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, self.backgroundTexture)

# draw
glDrawArrays(GL_TRIANGLES, 0, 6)

No need to rebind our buffers or specify our vertex shader inputs.

Ciao!

P.S.

Here’s the complete Stereo Depth class, making use of a Vertex Array Object:

from OpenGL.GL import *
from OpenGL.GLUT import *
from OpenGL.GL.shaders import *
import numpy, math
from PIL import Image

class StereoDepth:
 
    # constants
    BACKGROUND_IMAGE = 'image_left.png'
 
    # vertex shader program
    vertexShader = """
        #version 330 core
     
        attribute vec3 vert;
        attribute vec2 uV;
        uniform mat4 mvMatrix;
        uniform mat4 pMatrix;
        out vec2 UV;
     
        void main() {
          gl_Position = pMatrix * mvMatrix * vec4(vert, 1.0);
          UV = uV;
        }
    """
 
    # fragment shader program
    fragmentShader = """
        #version 330 core
     
        in vec2 UV;
        uniform sampler2D backgroundTexture;
        out vec3 colour;
     
        void main() {
          colour = texture(backgroundTexture, UV).rgb;
        }
    """

    # initialise opengl
    def _init_opengl(self):
 
        # create shader program
        vs = compileShader(self.vertexShader, GL_VERTEX_SHADER)
        fs = compileShader(self.fragmentShader, GL_FRAGMENT_SHADER)
        self.program = compileProgram(vs, fs)
        glUseProgram(self.program)
 
        # obtain uniforms and attributes
        aVert = glGetAttribLocation(self.program, "vert")
        aUV = glGetAttribLocation(self.program, "uV")
        self.uPMatrix = glGetUniformLocation(self.program, 'pMatrix')
        self.uMVMatrix = glGetUniformLocation(self.program, "mvMatrix")
        self.uBackgroundTexture = glGetUniformLocation(self.program, "backgroundTexture")
 
        # set background vertex
        backgroundVertex = [
            -2.0,  1.5, -3.6, 
            -2.0, -1.5, -3.6,
             2.0,  1.5, -3.6, 
             2.0,  1.5, -3.6, 
            -2.0, -1.5, -3.6, 
             2.0, -1.5, -3.6]
 
        # set background UV
        backgroundUV = [
            0.0, 0.0,
            0.0, 1.0,
            1.0, 0.0,
            1.0, 0.0,
            0.0, 1.0,
            1.0, 1.0]

        # create and bind VAO
        self.backgroundVAO = glGenVertexArrays(1)
        glBindVertexArray(self.backgroundVAO)

        # generate vertex buffer
        vertexBuffer = glGenBuffers(1)
        glBindBuffer(GL_ARRAY_BUFFER, vertexBuffer)

        # enable vertex attribute array
        glEnableVertexAttribArray(aVert)

        # vertex attribute pointer
        glVertexAttribPointer(aVert, 3, GL_FLOAT, GL_FALSE, 0, None)

        # buffer vertex data
        vertexData = numpy.array(backgroundVertex, numpy.float32)
        glBufferData(GL_ARRAY_BUFFER, 4 * len(vertexData), vertexData, GL_STATIC_DRAW)

        # generate UV buffer
        uvBuffer = glGenBuffers(1)
        glBindBuffer(GL_ARRAY_BUFFER, uvBuffer)

        # enable UV attribute array
        glEnableVertexAttribArray(aUV)

        # UV attribute pointer
        glVertexAttribPointer(aUV, 2, GL_FLOAT, GL_FALSE, 0, None)

        # buffer UV data
        uvData = numpy.array(backgroundUV, numpy.float32)
        glBufferData(GL_ARRAY_BUFFER, 4 * len(uvData), uvData, GL_STATIC_DRAW)

        # unbind and disable
        glBindVertexArray(0)
        glDisableVertexAttribArray(aVert)
        glDisableVertexAttribArray(aUV)
        glBindBuffer(GL_ARRAY_BUFFER, 0)

        # set background texture
        backgroundImage = Image.open(self.BACKGROUND_IMAGE)
        backgroundImageData = numpy.array(list(backgroundImage.getdata()), numpy.uint8)
         
        self.backgroundTexture = glGenTextures(1)
        glBindTexture(GL_TEXTURE_2D, self.backgroundTexture)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR)
        glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR)
        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, backgroundImage.size[0], backgroundImage.size[1], 0, GL_RGB, GL_UNSIGNED_BYTE, backgroundImageData)
        glBindTexture(GL_TEXTURE_2D, 0)

    # draw frame
    def _draw_frame(self):
 
        # create modelview matrix
        mvMatrix = numpy.array([
            1.0, 0.0, 0.0, 0.0,
            0.0, 1.0, 0.0, 0.0,
            0.0, 0.0, 1.0, 0.0,
            0.0, 0.0, 0.0, 1.0], numpy.float32)

        # create projection matrix
        fov = math.radians(45.0)
        f = 1.0 / math.tan(fov / 2.0)
        zNear = 0.1
        zFar = 100.0
        aspect = glutGet(GLUT_WINDOW_WIDTH) / float(glutGet(GLUT_WINDOW_HEIGHT))
        pMatrix = numpy.array([
            f / aspect, 0.0, 0.0, 0.0,
            0.0, f, 0.0, 0.0,
            0.0, 0.0, (zFar + zNear) / (zNear - zFar), -1.0,
            0.0, 0.0, 2.0 * zFar * zNear / (zNear - zFar), 0.0], numpy.float32)

        # use shader program
        glUseProgram(self.program)
 
        # set uniforms
        glUniformMatrix4fv(self.uMVMatrix, 1, GL_FALSE, mvMatrix)
        glUniformMatrix4fv(self.uPMatrix, 1, GL_FALSE, pMatrix)
        glUniform1i(self.uBackgroundTexture, 0)
  
        # bind VAO
        glBindVertexArray(self.backgroundVAO)

        # bind background texture
        glActiveTexture(GL_TEXTURE0);
        glBindTexture(GL_TEXTURE_2D, self.backgroundTexture)

        # draw
        glDrawArrays(GL_TRIANGLES, 0, 6)

        # unbind
        glBindVertexArray(0)
        glUseProgram(0)

        # swap buffers
        glutSwapBuffers()

    # setup and run OpenGL
    def main(self):
        glutInit()
        glutInitWindowSize(640, 480)
        glutInitWindowPosition(100, 100)
        glutCreateWindow('Stereo Depth')
        glutDisplayFunc(self._draw_frame)
        self._init_opengl()
        glutMainLoop()
 
# run an instance of StereoDepth
StereoDepth().main()
Advertisements