Add GL canvas wrapper
This commit is contained in:
parent
6c92837317
commit
e56d32180c
23
package-lock.json
generated
23
package-lock.json
generated
@ -3796,6 +3796,16 @@
|
||||
"integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==",
|
||||
"dev": true
|
||||
},
|
||||
"bindings": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
|
||||
"integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"file-uri-to-path": "1.0.0"
|
||||
}
|
||||
},
|
||||
"bl": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz",
|
||||
@ -6811,6 +6821,13 @@
|
||||
"integrity": "sha512-qyQ0pzAy78gVoJsmYeNgl8uH8yKhr1lVhW7JbzJmnlRi0I4R2eEDEJZVKG8agpDnLpacwNbDhLNG/LMdxHD2YQ==",
|
||||
"dev": true
|
||||
},
|
||||
"file-uri-to-path": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
|
||||
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
|
||||
"dev": true,
|
||||
"optional": true
|
||||
},
|
||||
"filename-reserved-regex": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-2.0.0.tgz",
|
||||
@ -7485,6 +7502,11 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"gl-matrix": {
|
||||
"version": "3.4.3",
|
||||
"resolved": "https://registry.npmjs.org/gl-matrix/-/gl-matrix-3.4.3.tgz",
|
||||
"integrity": "sha512-wcCp8vu8FT22BnvKVPjXa/ICBWRq/zjFfdofZy1WSpQZpphblv12/bOQLBC1rMM7SGOFS9ltVmKOHil5+Ml7gA=="
|
||||
},
|
||||
"glob": {
|
||||
"version": "7.2.3",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
|
||||
@ -16019,6 +16041,7 @@
|
||||
"dev": true,
|
||||
"optional": true,
|
||||
"requires": {
|
||||
"bindings": "^1.5.0",
|
||||
"nan": "^2.12.1"
|
||||
}
|
||||
},
|
||||
|
||||
@ -31,6 +31,7 @@
|
||||
"@types/resize-observer-browser": "^0.1.6",
|
||||
"concurrently": "^5.3.0",
|
||||
"fs-extra": "^7.0.1",
|
||||
"gl-matrix": "^3.4.3",
|
||||
"json-cyclic": "0.0.3",
|
||||
"lodash": "^4.17.21",
|
||||
"mdi-vue": "^3.0.11",
|
||||
@ -72,8 +73,6 @@
|
||||
"webpack": "^4.46.0",
|
||||
"webpack-chrome-extension-reloader": "^0.8.3",
|
||||
"webpack-cli": "^3.3.12",
|
||||
"webpack-shell-plugin": "^0.5.0",
|
||||
"@vue/cli": "^4.5.15",
|
||||
"@vue/cli-plugin-typescript": "^4.5.15"
|
||||
"webpack-shell-plugin": "^0.5.0"
|
||||
}
|
||||
}
|
||||
|
||||
326
src/ext/lib/aard/gl/GlCanvas.ts
Normal file
326
src/ext/lib/aard/gl/GlCanvas.ts
Normal file
@ -0,0 +1,326 @@
|
||||
import { mat4 } from 'gl-matrix';
|
||||
import { GlCanvasBuffers, initBuffers } from './gl-init';
|
||||
|
||||
export interface GlCanvasOptions {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
// Vertex shader program
|
||||
const vsSource = `
|
||||
attribute vec4 aVertexPosition;
|
||||
attribute vec3 aVertexNormal;
|
||||
attribute vec2 aTextureCoord;
|
||||
|
||||
uniform mat4 uNormalMatrix;
|
||||
uniform mat4 uModelViewMatrix;
|
||||
uniform mat4 uProjectionMatrix;
|
||||
|
||||
varying highp vec2 vTextureCoord;
|
||||
|
||||
void main(void) {
|
||||
gl_Position = uProjectionMatrix * aVertexPosition;
|
||||
vTextureCoord = aTextureCoord;
|
||||
}
|
||||
`;
|
||||
// Fragment shader program
|
||||
const fsSource = `
|
||||
varying highp vec2 vTextureCoord;
|
||||
uniform sampler2D uSampler;
|
||||
|
||||
void main(void) {
|
||||
highp vec4 texelColor = texture2D(uSampler, vTextureCoord);
|
||||
|
||||
gl_FragColor = vec4(texelColor.rgb, texelColor.a);
|
||||
}
|
||||
`;
|
||||
|
||||
interface GlCanvasProgramInfo {
|
||||
program: WebGLProgram;
|
||||
attribLocations: {
|
||||
vertexPosition: number;
|
||||
vertexNormal: number;
|
||||
textureCoord: number;
|
||||
};
|
||||
uniformLocations: {
|
||||
projectionMatrix: WebGLUniformLocation;
|
||||
modelViewMatrix: WebGLUniformLocation;
|
||||
normalMatrix: WebGLUniformLocation;
|
||||
uSampler: WebGLUniformLocation;
|
||||
};
|
||||
}
|
||||
|
||||
export class GlCanvas {
|
||||
|
||||
private _canvas: HTMLCanvasElement;
|
||||
private set canvas(x: HTMLCanvasElement) {
|
||||
this._canvas = x;
|
||||
};
|
||||
public get canvas(): HTMLCanvasElement {
|
||||
return this._canvas;
|
||||
};
|
||||
|
||||
private _context: WebGLRenderingContext;
|
||||
private set gl(x: WebGLRenderingContext) {
|
||||
this._context = x;
|
||||
};
|
||||
private get gl(): WebGLRenderingContext {
|
||||
return this._context;
|
||||
}
|
||||
|
||||
private frameBufferSize: number;
|
||||
private _frameBuffer: Uint8Array;
|
||||
private set frameBuffer(x: Uint8Array) {
|
||||
this._frameBuffer = x;
|
||||
}
|
||||
public get frameBuffer(): Uint8Array {
|
||||
return this._frameBuffer;
|
||||
}
|
||||
|
||||
private buffers: GlCanvasBuffers;
|
||||
private texture: WebGLTexture;
|
||||
private programInfo: GlCanvasProgramInfo;
|
||||
private projectionMatrix: mat4;
|
||||
|
||||
|
||||
constructor(options: GlCanvasOptions) {
|
||||
this.canvas = document.createElement('canvas');
|
||||
this.gl = this.canvas.getContext('webgl');
|
||||
|
||||
if (!this.gl) {
|
||||
throw new Error('WebGL not supported');
|
||||
}
|
||||
|
||||
this.canvas.width = options.width;
|
||||
this.canvas.height = options.height;
|
||||
|
||||
this.frameBufferSize = options.width * options.height * 4;
|
||||
|
||||
this.initWebgl();
|
||||
}
|
||||
|
||||
/**
|
||||
* Draws video frame to the GL canvas
|
||||
* @param video video to extract a frame from
|
||||
*/
|
||||
drawVideoFrame(video: HTMLVideoElement): void {
|
||||
this.updateTexture(video);
|
||||
this.drawScene();
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads pixels from the canvas
|
||||
* @returns
|
||||
*/
|
||||
getImageData(): Uint8Array {
|
||||
this.gl.readPixels(0, 0, this.canvas.width, this.canvas.height, this.gl.RGBA, this.gl.UNSIGNED_BYTE, this.frameBuffer);
|
||||
return this.frameBuffer;
|
||||
}
|
||||
|
||||
private initWebgl() {
|
||||
// Initialize the GL context
|
||||
this.gl.clearColor(0.0, 0.0, 0.0, 1.0);
|
||||
this.gl.clear(this.gl.COLOR_BUFFER_BIT);
|
||||
|
||||
// Create shader program
|
||||
const shaderProgram = this.initShaderProgram();
|
||||
|
||||
// Setup params for shader program
|
||||
this.programInfo = {
|
||||
program: shaderProgram,
|
||||
attribLocations: {
|
||||
vertexPosition: this.gl.getAttribLocation(shaderProgram, "aVertexPosition"),
|
||||
vertexNormal: this.gl.getAttribLocation(shaderProgram, "aVertexNormal"),
|
||||
textureCoord: this.gl.getAttribLocation(shaderProgram, "aTextureCoord"),
|
||||
},
|
||||
uniformLocations: {
|
||||
projectionMatrix: this.gl.getUniformLocation(
|
||||
shaderProgram,
|
||||
"uProjectionMatrix"
|
||||
),
|
||||
modelViewMatrix: this.gl.getUniformLocation(shaderProgram, "uModelViewMatrix"),
|
||||
normalMatrix: this.gl.getUniformLocation(shaderProgram, "uNormalMatrix"),
|
||||
uSampler: this.gl.getUniformLocation(shaderProgram, "uSampler"),
|
||||
},
|
||||
};
|
||||
|
||||
// Here's where we call the routine that builds all the
|
||||
// objects we'll be drawing.
|
||||
this.buffers = initBuffers(this.gl);
|
||||
this.initTexture();
|
||||
|
||||
// Flip image pixels into the bottom-to-top order that WebGL expects.
|
||||
this.gl.pixelStorei(this.gl.UNPACK_FLIP_Y_WEBGL, true);
|
||||
|
||||
|
||||
// Since our matrix is never going to change, we can define projection outside of drawScene function:
|
||||
this.projectionMatrix = mat4.create();
|
||||
mat4.ortho(this.projectionMatrix, -1, 1, -1, 1, -10, 10);
|
||||
|
||||
// we will be reusing our frame buffer for all draws and reads
|
||||
// this improves performance and lessens production of garbage,
|
||||
// translating into fewer garbage collections (probably), resulting
|
||||
// in fewer hitches and other performance issues (probably)
|
||||
this.frameBuffer = new Uint8Array(this.frameBufferSize);
|
||||
}
|
||||
|
||||
private loadShader(type, source) {
|
||||
const shader = this.gl.createShader(type);
|
||||
this.gl.shaderSource(shader, source);
|
||||
this.gl.compileShader(shader);
|
||||
|
||||
// TODO: warn if shader failed to compile
|
||||
if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) {
|
||||
this.gl.deleteShader(shader);
|
||||
return null;
|
||||
}
|
||||
|
||||
return shader;
|
||||
}
|
||||
|
||||
private initShaderProgram() {
|
||||
const vertexShader = this.loadShader(this.gl.VERTEX_SHADER, vsSource);
|
||||
const fragmentShader = this.loadShader(this.gl.FRAGMENT_SHADER, fsSource);
|
||||
|
||||
// Create the shader program
|
||||
const shaderProgram = this.gl.createProgram();
|
||||
this.gl.attachShader(shaderProgram, vertexShader);
|
||||
this.gl.attachShader(shaderProgram, fragmentShader);
|
||||
this.gl.linkProgram(shaderProgram);
|
||||
|
||||
// TODO: maybe give a warning if program failed to initialize
|
||||
if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return shaderProgram;
|
||||
}
|
||||
|
||||
private initTexture(): void {
|
||||
this.texture = this.gl.createTexture();
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture);
|
||||
|
||||
// Because video has to be download over the internet
|
||||
// they might take a moment until it's ready so
|
||||
// put a single pixel in the texture so we can
|
||||
// use it immediately.
|
||||
const level = 0;
|
||||
const internalFormat = this.gl.RGBA;
|
||||
const width = 1;
|
||||
const height = 1;
|
||||
const border = 0;
|
||||
const srcFormat = this.gl.RGBA;
|
||||
const srcType = this.gl.UNSIGNED_BYTE;
|
||||
const pixel = new Uint8Array([0, 0, 255, 255]); // opaque blue
|
||||
this.gl.texImage2D(
|
||||
this.gl.TEXTURE_2D,
|
||||
level,
|
||||
internalFormat,
|
||||
width,
|
||||
height,
|
||||
border,
|
||||
srcFormat,
|
||||
srcType,
|
||||
pixel
|
||||
);
|
||||
|
||||
// Turn off mips and set wrapping to clamp to edge so it
|
||||
// will work regardless of the dimensions of the video.
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
|
||||
this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR);
|
||||
}
|
||||
|
||||
private updateTexture(video: HTMLVideoElement) {
|
||||
const level = 0;
|
||||
const internalFormat = this.gl.RGBA;
|
||||
const srcFormat = this.gl.RGBA;
|
||||
const srcType = this.gl.UNSIGNED_BYTE;
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture);
|
||||
this.gl.texImage2D(
|
||||
this.gl.TEXTURE_2D,
|
||||
level,
|
||||
internalFormat,
|
||||
srcFormat,
|
||||
srcType,
|
||||
video
|
||||
);
|
||||
}
|
||||
|
||||
private setTextureAttribute() {
|
||||
const num = 2; // every coordinate composed of 2 values
|
||||
const type = this.gl.FLOAT; // the data in the buffer is 32-bit float
|
||||
const normalize = false; // don't normalize
|
||||
const stride = 0; // how many bytes to get from one set to the next
|
||||
const offset = 0; // how many bytes inside the buffer to start from
|
||||
|
||||
this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.buffers.textureCoord);
|
||||
this.gl.vertexAttribPointer(
|
||||
this.programInfo.attribLocations.textureCoord,
|
||||
num,
|
||||
type,
|
||||
normalize,
|
||||
stride,
|
||||
offset
|
||||
);
|
||||
this.gl.enableVertexAttribArray(this.programInfo.attribLocations.textureCoord);
|
||||
}
|
||||
|
||||
private setPositionAttribute() {
|
||||
const numComponents = 3;
|
||||
const type = this.gl.FLOAT; // the data in the buffer is 32bit floats
|
||||
const normalize = false; // don't normalize
|
||||
const stride = 0; // how many bytes to get from one set of values to the next
|
||||
// 0 = use type and numComponents above
|
||||
const offset = 0; // how many bytes inside the buffer to start from
|
||||
this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.buffers.position);
|
||||
this.gl.vertexAttribPointer(
|
||||
this.programInfo.attribLocations.vertexPosition,
|
||||
numComponents,
|
||||
type,
|
||||
normalize,
|
||||
stride,
|
||||
offset
|
||||
);
|
||||
this.gl.enableVertexAttribArray(this.programInfo.attribLocations.vertexPosition);
|
||||
}
|
||||
|
||||
private drawScene(): void {
|
||||
/**
|
||||
* Since we are drawing our frames in a way such that the entire canvas is
|
||||
* always covered by rendered video, and given our video is the only object
|
||||
* being rendered to the canvas, we can avoid some things, such as:
|
||||
* * clearing the canvas
|
||||
* * any sort of depth tests
|
||||
*/
|
||||
|
||||
// Tell WebGL how to pull out the positions from the position
|
||||
// buffer into the vertexPosition attribute.
|
||||
this.setPositionAttribute();
|
||||
this.setTextureAttribute();
|
||||
|
||||
// Tell WebGL which indices to use to index the vertices, and to use
|
||||
// our program when drawing video frame to the canvas
|
||||
this.gl.bindBuffer(this.gl.ELEMENT_ARRAY_BUFFER, this.buffers.indices);
|
||||
this.gl.useProgram(this.programInfo.program);
|
||||
|
||||
// Set the shader uniforms
|
||||
this.gl.uniformMatrix4fv(
|
||||
this.programInfo.uniformLocations.projectionMatrix,
|
||||
false,
|
||||
this.projectionMatrix
|
||||
);
|
||||
|
||||
// Tell WebGL we want to affect texture unit 0, bind texture to texture unit 0,
|
||||
// and tell the shader that we bound the texture to texture unit 0.
|
||||
this.gl.activeTexture(this.gl.TEXTURE0);
|
||||
this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture);
|
||||
this.gl.uniform1i(this.programInfo.uniformLocations.uSampler, 0);
|
||||
|
||||
// draw geometry
|
||||
const vertexCount = 6;
|
||||
const type = this.gl.UNSIGNED_SHORT;
|
||||
const offset = 0;
|
||||
this.gl.drawElements(this.gl.TRIANGLES, vertexCount, type, offset);
|
||||
}
|
||||
}
|
||||
84
src/ext/lib/aard/gl/gl-init.ts
Normal file
84
src/ext/lib/aard/gl/gl-init.ts
Normal file
@ -0,0 +1,84 @@
|
||||
export interface GlCanvasBuffers {
|
||||
position: WebGLBuffer,
|
||||
normal: WebGLBuffer,
|
||||
textureCoord: WebGLBuffer,
|
||||
indices: WebGLBuffer,
|
||||
};
|
||||
|
||||
export function initBuffers(gl: WebGLRenderingContext): GlCanvasBuffers {
|
||||
const positionBuffer = initPositionBuffer(gl);
|
||||
const textureCoordBuffer = initTextureBuffer(gl);
|
||||
const indexBuffer = initIndexBuffer(gl);
|
||||
const normalBuffer = initNormalBuffer(gl);
|
||||
|
||||
return {
|
||||
position: positionBuffer,
|
||||
normal: normalBuffer,
|
||||
textureCoord: textureCoordBuffer,
|
||||
indices: indexBuffer,
|
||||
};
|
||||
}
|
||||
|
||||
function initPositionBuffer(gl: WebGLRenderingContext) {
|
||||
const positionBuffer = gl.createBuffer();
|
||||
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);
|
||||
const positions = [
|
||||
-1.0, -1.0, 1.0, 1.0, -1.0, 1.0, 1.0, 1.0, 1.0, -1.0, 1.0, 1.0,
|
||||
];
|
||||
gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW);
|
||||
|
||||
return positionBuffer;
|
||||
}
|
||||
|
||||
function initIndexBuffer(gl: WebGLRenderingContext) {
|
||||
const indexBuffer = gl.createBuffer();
|
||||
gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, indexBuffer);
|
||||
|
||||
const indices = [
|
||||
0, 1, 2,
|
||||
0, 2, 3,
|
||||
];
|
||||
|
||||
gl.bufferData(
|
||||
gl.ELEMENT_ARRAY_BUFFER,
|
||||
new Uint16Array(indices),
|
||||
gl.STATIC_DRAW
|
||||
);
|
||||
|
||||
return indexBuffer;
|
||||
}
|
||||
|
||||
function initTextureBuffer(gl: WebGLRenderingContext) {
|
||||
const textureCoordBuffer = gl.createBuffer();
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, textureCoordBuffer);
|
||||
|
||||
const textureCoordinates = [
|
||||
0.0, 0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0,
|
||||
];
|
||||
|
||||
gl.bufferData(
|
||||
gl.ARRAY_BUFFER,
|
||||
new Float32Array(textureCoordinates),
|
||||
gl.STATIC_DRAW
|
||||
);
|
||||
|
||||
return textureCoordBuffer;
|
||||
}
|
||||
|
||||
function initNormalBuffer(gl: WebGLRenderingContext) {
|
||||
const normalBuffer = gl.createBuffer();
|
||||
gl.bindBuffer(gl.ARRAY_BUFFER, normalBuffer);
|
||||
|
||||
const vertexNormals = [
|
||||
0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0,
|
||||
];
|
||||
|
||||
gl.bufferData(
|
||||
gl.ARRAY_BUFFER,
|
||||
new Float32Array(vertexNormals),
|
||||
gl.STATIC_DRAW
|
||||
);
|
||||
|
||||
return normalBuffer;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user