diff --git a/package-lock.json b/package-lock.json index f26d5e2..453ff6c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3796,6 +3796,16 @@ "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", "dev": true }, + "bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dev": true, + "optional": true, + "requires": { + "file-uri-to-path": "1.0.0" + } + }, "bl": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", @@ -6811,6 +6821,13 @@ "integrity": "sha512-qyQ0pzAy78gVoJsmYeNgl8uH8yKhr1lVhW7JbzJmnlRi0I4R2eEDEJZVKG8agpDnLpacwNbDhLNG/LMdxHD2YQ==", "dev": true }, + "file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "dev": true, + "optional": true + }, "filename-reserved-regex": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-2.0.0.tgz", @@ -7485,6 +7502,11 @@ } } }, + "gl-matrix": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/gl-matrix/-/gl-matrix-3.4.3.tgz", + "integrity": "sha512-wcCp8vu8FT22BnvKVPjXa/ICBWRq/zjFfdofZy1WSpQZpphblv12/bOQLBC1rMM7SGOFS9ltVmKOHil5+Ml7gA==" + }, "glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -16019,6 +16041,7 @@ "dev": true, "optional": true, "requires": { + "bindings": "^1.5.0", "nan": "^2.12.1" } }, diff --git a/package.json b/package.json index e811f85..8b4745b 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "@types/resize-observer-browser": "^0.1.6", "concurrently": "^5.3.0", "fs-extra": "^7.0.1", + "gl-matrix": "^3.4.3", "json-cyclic": "0.0.3", "lodash": "^4.17.21", "mdi-vue": "^3.0.11", @@ -72,8 +73,6 @@ "webpack": "^4.46.0", "webpack-chrome-extension-reloader": "^0.8.3", "webpack-cli": "^3.3.12", - "webpack-shell-plugin": "^0.5.0", - "@vue/cli": "^4.5.15", - "@vue/cli-plugin-typescript": "^4.5.15" + "webpack-shell-plugin": "^0.5.0" } } diff --git a/src/ext/lib/aard/gl/GlCanvas.ts b/src/ext/lib/aard/gl/GlCanvas.ts new file mode 100644 index 0000000..6ec954d --- /dev/null +++ b/src/ext/lib/aard/gl/GlCanvas.ts @@ -0,0 +1,326 @@ +import { mat4 } from 'gl-matrix'; +import { GlCanvasBuffers, initBuffers } from './gl-init'; + +export interface GlCanvasOptions { + width: number; + height: number; +} + +// Vertex shader program +const vsSource = ` + attribute vec4 aVertexPosition; + attribute vec3 aVertexNormal; + attribute vec2 aTextureCoord; + + uniform mat4 uNormalMatrix; + uniform mat4 uModelViewMatrix; + uniform mat4 uProjectionMatrix; + + varying highp vec2 vTextureCoord; + + void main(void) { + gl_Position = uProjectionMatrix * aVertexPosition; + vTextureCoord = aTextureCoord; + } +`; +// Fragment shader program +const fsSource = ` + varying highp vec2 vTextureCoord; + uniform sampler2D uSampler; + + void main(void) { + highp vec4 texelColor = texture2D(uSampler, vTextureCoord); + + gl_FragColor = vec4(texelColor.rgb, texelColor.a); + } +`; + +interface GlCanvasProgramInfo { + program: WebGLProgram; + attribLocations: { + vertexPosition: number; + vertexNormal: number; + textureCoord: number; + }; + uniformLocations: { + projectionMatrix: WebGLUniformLocation; + modelViewMatrix: WebGLUniformLocation; + normalMatrix: WebGLUniformLocation; + uSampler: WebGLUniformLocation; + }; +} + +export class GlCanvas { + + private _canvas: HTMLCanvasElement; + private set canvas(x: HTMLCanvasElement) { + this._canvas = x; + }; + public get canvas(): HTMLCanvasElement { + return this._canvas; + }; + + private _context: WebGLRenderingContext; + private set gl(x: WebGLRenderingContext) { + this._context = x; + }; + private get gl(): WebGLRenderingContext { + return this._context; + } + + private frameBufferSize: number; + private _frameBuffer: Uint8Array; + private set frameBuffer(x: Uint8Array) { + this._frameBuffer = x; + } + public get frameBuffer(): Uint8Array { + return this._frameBuffer; + } + + private buffers: GlCanvasBuffers; + private texture: WebGLTexture; + private programInfo: GlCanvasProgramInfo; + private projectionMatrix: mat4; + + + constructor(options: GlCanvasOptions) { + this.canvas = document.createElement('canvas'); + this.gl = this.canvas.getContext('webgl'); + + if (!this.gl) { + throw new Error('WebGL not supported'); + } + + this.canvas.width = options.width; + this.canvas.height = options.height; + + this.frameBufferSize = options.width * options.height * 4; + + this.initWebgl(); + } + + /** + * Draws video frame to the GL canvas + * @param video video to extract a frame from + */ + drawVideoFrame(video: HTMLVideoElement): void { + this.updateTexture(video); + this.drawScene(); + } + + /** + * Reads pixels from the canvas + * @returns + */ + getImageData(): Uint8Array { + this.gl.readPixels(0, 0, this.canvas.width, this.canvas.height, this.gl.RGBA, this.gl.UNSIGNED_BYTE, this.frameBuffer); + return this.frameBuffer; + } + + private initWebgl() { + // Initialize the GL context + this.gl.clearColor(0.0, 0.0, 0.0, 1.0); + this.gl.clear(this.gl.COLOR_BUFFER_BIT); + + // Create shader program + const shaderProgram = this.initShaderProgram(); + + // Setup params for shader program + this.programInfo = { + program: shaderProgram, + attribLocations: { + vertexPosition: this.gl.getAttribLocation(shaderProgram, "aVertexPosition"), + vertexNormal: this.gl.getAttribLocation(shaderProgram, "aVertexNormal"), + textureCoord: this.gl.getAttribLocation(shaderProgram, "aTextureCoord"), + }, + uniformLocations: { + projectionMatrix: this.gl.getUniformLocation( + shaderProgram, + "uProjectionMatrix" + ), + modelViewMatrix: this.gl.getUniformLocation(shaderProgram, "uModelViewMatrix"), + normalMatrix: this.gl.getUniformLocation(shaderProgram, "uNormalMatrix"), + uSampler: this.gl.getUniformLocation(shaderProgram, "uSampler"), + }, + }; + + // Here's where we call the routine that builds all the + // objects we'll be drawing. + this.buffers = initBuffers(this.gl); + this.initTexture(); + + // Flip image pixels into the bottom-to-top order that WebGL expects. + this.gl.pixelStorei(this.gl.UNPACK_FLIP_Y_WEBGL, true); + + + // Since our matrix is never going to change, we can define projection outside of drawScene function: + this.projectionMatrix = mat4.create(); + mat4.ortho(this.projectionMatrix, -1, 1, -1, 1, -10, 10); + + // we will be reusing our frame buffer for all draws and reads + // this improves performance and lessens production of garbage, + // translating into fewer garbage collections (probably), resulting + // in fewer hitches and other performance issues (probably) + this.frameBuffer = new Uint8Array(this.frameBufferSize); + } + + private loadShader(type, source) { + const shader = this.gl.createShader(type); + this.gl.shaderSource(shader, source); + this.gl.compileShader(shader); + + // TODO: warn if shader failed to compile + if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) { + this.gl.deleteShader(shader); + return null; + } + + return shader; + } + + private initShaderProgram() { + const vertexShader = this.loadShader(this.gl.VERTEX_SHADER, vsSource); + const fragmentShader = this.loadShader(this.gl.FRAGMENT_SHADER, fsSource); + + // Create the shader program + const shaderProgram = this.gl.createProgram(); + this.gl.attachShader(shaderProgram, vertexShader); + this.gl.attachShader(shaderProgram, fragmentShader); + this.gl.linkProgram(shaderProgram); + + // TODO: maybe give a warning if program failed to initialize + if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) { + return null; + } + + return shaderProgram; + } + + private initTexture(): void { + this.texture = this.gl.createTexture(); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture); + + // Because video has to be download over the internet + // they might take a moment until it's ready so + // put a single pixel in the texture so we can + // use it immediately. + const level = 0; + const internalFormat = this.gl.RGBA; + const width = 1; + const height = 1; + const border = 0; + const srcFormat = this.gl.RGBA; + const srcType = this.gl.UNSIGNED_BYTE; + const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue + this.gl.texImage2D( + this.gl.TEXTURE_2D, + level, + internalFormat, + width, + height, + border, + srcFormat, + srcType, + pixel + ); + + // Turn off mips and set wrapping to clamp to edge so it + // will work regardless of the dimensions of the video. + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR); + } + + private updateTexture(video: HTMLVideoElement) { + const level = 0; + const internalFormat = this.gl.RGBA; + const srcFormat = this.gl.RGBA; + const srcType = this.gl.UNSIGNED_BYTE; + this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture); + this.gl.texImage2D( + this.gl.TEXTURE_2D, + level, + internalFormat, + srcFormat, + srcType, + video + ); + } + + private setTextureAttribute() { + const num = 2; // every coordinate composed of 2 values + const type = this.gl.FLOAT; // the data in the buffer is 32-bit float + const normalize = false; // don't normalize + const stride = 0; // how many bytes to get from one set to the next + const offset = 0; // how many bytes inside the buffer to start from + + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.buffers.textureCoord); + this.gl.vertexAttribPointer( + this.programInfo.attribLocations.textureCoord, + num, + type, + normalize, + stride, + offset + ); + this.gl.enableVertexAttribArray(this.programInfo.attribLocations.textureCoord); + } + + private setPositionAttribute() { + const numComponents = 3; + const type = this.gl.FLOAT; // the data in the buffer is 32bit floats + const normalize = false; // don't normalize + const stride = 0; // how many bytes to get from one set of values to the next + // 0 = use type and numComponents above + const offset = 0; // how many bytes inside the buffer to start from + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.buffers.position); + this.gl.vertexAttribPointer( + this.programInfo.attribLocations.vertexPosition, + numComponents, + type, + normalize, + stride, + offset + ); + this.gl.enableVertexAttribArray(this.programInfo.attribLocations.vertexPosition); + } + + private drawScene(): void { + /** + * Since we are drawing our frames in a way such that the entire canvas is + * always covered by rendered video, and given our video is the only object + * being rendered to the canvas, we can avoid some things, such as: + * * clearing the canvas + * * any sort of depth tests + */ + + // Tell WebGL how to pull out the positions from the position + // buffer into the vertexPosition attribute. + this.setPositionAttribute(); + this.setTextureAttribute(); + + // Tell WebGL which indices to use to index the vertices, and to use + // our program when drawing video frame to the canvas + this.gl.bindBuffer(this.gl.ELEMENT_ARRAY_BUFFER, this.buffers.indices); + this.gl.useProgram(this.programInfo.program); + + // Set the shader uniforms + this.gl.uniformMatrix4fv( + this.programInfo.uniformLocations.projectionMatrix, + false, + this.projectionMatrix + ); + + // Tell WebGL we want to affect texture unit 0, bind texture to texture unit 0, + // and tell the shader that we bound the texture to texture unit 0. + this.gl.activeTexture(this.gl.TEXTURE0); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture); + this.gl.uniform1i(this.programInfo.uniformLocations.uSampler, 0); + + // draw geometry + const vertexCount = 6; + const type = this.gl.UNSIGNED_SHORT; + const offset = 0; + this.gl.drawElements(this.gl.TRIANGLES, vertexCount, type, offset); + } +} diff --git a/src/ext/lib/aard/gl/gl-init.ts b/src/ext/lib/aard/gl/gl-init.ts new file mode 100644 index 0000000..cae1727 --- /dev/null +++ b/src/ext/lib/aard/gl/gl-init.ts @@ -0,0 +1,84 @@ +export interface GlCanvasBuffers { + position: WebGLBuffer, + normal: WebGLBuffer, + textureCoord: WebGLBuffer, + indices: WebGLBuffer, +}; + +export function initBuffers(gl: WebGLRenderingContext): GlCanvasBuffers { + const positionBuffer = initPositionBuffer(gl); + const textureCoordBuffer = initTextureBuffer(gl); + const indexBuffer = initIndexBuffer(gl); + const normalBuffer = initNormalBuffer(gl); + + return { + position: positionBuffer, + normal: normalBuffer, + textureCoord: textureCoordBuffer, + indices: indexBuffer, + }; +} + +function initPositionBuffer(gl: WebGLRenderingContext) { + const positionBuffer = gl.createBuffer(); + + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + const positions = [ + -1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0, + ]; + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW); + + return positionBuffer; +} + +function initIndexBuffer(gl: WebGLRenderingContext) { + const indexBuffer = gl.createBuffer(); + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, indexBuffer); + + const indices = [ + 0, 1, 2, + 0, 2, 3, + ]; + + gl.bufferData( + gl.ELEMENT_ARRAY_BUFFER, + new Uint16Array(indices), + gl.STATIC_DRAW + ); + + return indexBuffer; +} + +function initTextureBuffer(gl: WebGLRenderingContext) { + const textureCoordBuffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, textureCoordBuffer); + + const textureCoordinates = [ + 0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, + ]; + + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array(textureCoordinates), + gl.STATIC_DRAW + ); + + return textureCoordBuffer; +} + +function initNormalBuffer(gl: WebGLRenderingContext) { + const normalBuffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, normalBuffer); + + const vertexNormals = [ + 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, + ]; + + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array(vertexNormals), + gl.STATIC_DRAW + ); + + return normalBuffer; +}