Switch to gl canvas

src/ext/lib/aard/gl/GlCanvas.ts

@@ -82,6 +82,12 @@ export class GlCanvas {
   private programInfo: GlCanvasProgramInfo;
   private projectionMatrix: mat4;
 
+  get width() {
+    return this.canvas.width;
+  }
+  get height() {
+    return this.canvas.height;
+  }
 
   constructor(options: GlCanvasOptions) {
     this.canvas = document.createElement('canvas');
@@ -117,6 +123,18 @@ export class GlCanvas {
     return this.frameBuffer;
   }
 
+  /**
+   * Cleans up after itself
+   */
+  destroy() {
+    this.gl.deleteProgram(this.programInfo.program);
+    this.gl.deleteBuffer(this.buffers.position);
+    this.gl.deleteBuffer(this.buffers.normal);
+    this.gl.deleteBuffer(this.buffers.textureCoord);
+    this.gl.deleteBuffer(this.buffers.indices);
+    this.gl.deleteTexture(this.texture);
+  }
+
   private initWebgl() {
     // Initialize the GL context
     this.gl.clearColor(0.0, 0.0, 0.0, 1.0);

src/ext/lib/ar-detect/ArDetector.ts

@@ -14,6 +14,7 @@ import Logger from '../Logger';
 import VideoData from '../video-data/VideoData';
 import Settings from '../Settings';
 import EventBus from '../EventBus';
+import { GlCanvas } from '../aard/gl/GlCanvas';
 
 enum VideoPlaybackState {
   Playing,
@@ -82,11 +83,10 @@ class ArDetector {
   _nextTick: boolean;
 
   private animationFrameHandle: any;
-  private attachedCanvas: HTMLCanvasElement;
-  canvas: HTMLCanvasElement;
-  private context: CanvasRenderingContext2D;
   canvasImageDataRowLength: number;
 
+  glCanvas: GlCanvas;
+
   private timers = {
     nextFrameCheckTime: Date.now()
   }
@@ -189,43 +189,24 @@ class ArDetector {
     //
     // [1] initiate canvases
     //
-
+    if (this.glCanvas) {
-    if (!cwidth) {
+      this.glCanvas.destroy();
-      cwidth = this.settings.active.arDetect.canvasDimensions.sampleCanvas.width;
-      cheight = this.settings.active.arDetect.canvasDimensions.sampleCanvas.height;
     }
-
+    this.glCanvas = new GlCanvas(this.settings.active.arDetect.canvasDimensions.sampleCanvas);
-    if (this.canvas) {
-      this.canvas.remove();
-    }
-    // if (this.blackframeCanvas) {
-    //   this.blackframeCanvas.remove();
-    // }
-
-    // things to note: we'll be keeping canvas in memory only.
-    this.canvas = document.createElement("canvas");
-    this.canvas.width = cwidth;
-    this.canvas.height = cheight;
-    // this.blackframeCanvas = document.createElement("canvas");
-    // this.blackframeCanvas.width = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.width;
-    // this.blackframeCanvas.height = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.height;
-
-    this.context = this.canvas.getContext("2d");
 
     //
     // [2] determine places we'll use to sample our main frame
     //
-
     let ncol = this.settings.active.arDetect.sampling.staticCols;
     let nrow = this.settings.active.arDetect.sampling.staticRows;
 
-    let colSpacing = this.canvas.width / ncol;
+    let colSpacing = this.glCanvas.width / ncol;
-    let rowSpacing = (this.canvas.height << 2) / nrow;
+    let rowSpacing = (this.glCanvas.height << 2) / nrow;
 
     this.sampleLines = [];
     this.sampleCols = [];
 
-    for(let i = 0; i < ncol; i++){
+    for (let i = 0; i < ncol; i++){
       if(i < ncol - 1)
         this.sampleCols.push(Math.round(colSpacing * i));
       else{
@@ -233,7 +214,7 @@ class ArDetector {
       }
     }
 
-    for(let i = 0; i < nrow; i++){
+    for (let i = 0; i < nrow; i++){
       if(i < ncol - 5)
         this.sampleLines.push(Math.round(rowSpacing * i));
       else{
@@ -244,7 +225,6 @@ class ArDetector {
     //
     // [3] do other things setup needs to do
     //
-
     this.resetBlackLevel();
 
     // if we're restarting ArDetect, we need to do this in order to force-recalculate aspect ratio
@@ -253,11 +233,10 @@ class ArDetector {
     this.canvasImageDataRowLength = cwidth << 2;
 
     this.start();
-
+    // if(Debug.debugCanvas.enabled){
-    if(Debug.debugCanvas.enabled){
       // this.debugCanvas.init({width: cwidth, height: cheight});
       // DebugCanvas.draw("test marker","test","rect", {x:5, y:5}, {width: 5, height: 5});
-    }
+    // }
 
     this.conf.arSetupComplete = true;
   }
@@ -391,20 +370,6 @@ class ArDetector {
     );
   }
 
-  private attachCanvas(canvas){
-    if(this.attachedCanvas)
-      this.attachedCanvas.remove();
-
-    // todo: place canvas on top of the video instead of random location
-    canvas.style.position = "absolute";
-    canvas.style.left = "200px";
-    canvas.style.top = "1200px";
-    canvas.style.zIndex = 10000;
-
-    document.getElementsByTagName("body")[0]
-            .appendChild(canvas);
-  }
-
   /**
    * Adds execution time sample for performance metrics
    * @param performanceObject
@@ -749,19 +714,19 @@ class ArDetector {
     // aspect ratio and correct our calculations to account for that
 
     const fileAr = this.video.videoWidth / this.video.videoHeight;
-    const canvasAr = this.canvas.width / this.canvas.height;
+    const canvasAr = this.glCanvas.width / this.glCanvas.height;
     let widthCorrected;
 
     if (edges.top && edges.bottom) {
       // in case of letterbox, we take canvas height as canon and assume width got stretched or squished
 
       if (fileAr != canvasAr) {
-        widthCorrected = this.canvas.height * fileAr;
+        widthCorrected = this.glCanvas.height * fileAr;
       } else {
-        widthCorrected = this.canvas.width;
+        widthCorrected = this.glCanvas.width;
       }
 
-      return widthCorrected / (this.canvas.height - letterbox);
+      return widthCorrected / (this.glCanvas.height - letterbox);
     }
   }
 
@@ -822,22 +787,19 @@ class ArDetector {
       return;
     }
 
-    if (!this.context) {
+    if (!this.glCanvas) {
       this.init();
     }
 
     let sampleCols = this.sampleCols.slice(0);
 
-
-
     let startTime = performance.now();
-    await new Promise<void>(
+    const imageData = await new Promise<Uint8Array>(
       resolve => {
-        this.context.drawImage(this.video, 0, 0, this.canvas.width, this.canvas.height);
+        this.glCanvas.drawVideoFrame(this.video);
-        resolve();
+        resolve(this.glCanvas.getImageData());
       }
     )
-    const imageData = this.context.getImageData(0, 0, this.canvas.width, this.canvas.height).data;
     timerResults.imageDrawTime = performance.now() - startTime;
 
     startTime = performance.now();
@@ -1130,7 +1092,7 @@ class ArDetector {
     // returns 'false' if we found a non-black edge pixel.
 
     // If we detect anything darker than blackLevel, we modify blackLevel to the new lowest value
-    const rowOffset = this.canvas.width * (this.canvas.height - 1);
+    const rowOffset = this.glCanvas.width * (this.glCanvas.height - 1);
     let currentMin = 255, currentMax = 0, colOffset_r, colOffset_g, colOffset_b, colOffset_rb, colOffset_gb, colOffset_bb, blthreshold = this.settings.active.arDetect.blackbar.threshold;
 
     // detect black level. if currentMax comes above blackbar + blackbar threshold, we know we aren't letterboxed

src/ext/lib/ar-detect/GuardLine.ts

@@ -42,7 +42,7 @@ class GuardLine {
 
     // to odstrani vse neveljavne nastavitve in vse možnosti, ki niso smiselne
     // this removes any configs with invalid values or values that dont make sense
-    if (bbTop < 0 || bbBottom >= this.aard.canvas.height ){
+    if (bbTop < 0 || bbBottom >= this.aard.glCanvas.height ){
       throw {error: "INVALID_SETTINGS_IN_GUARDLINE", bbTop, bbBottom}
     }
 
@@ -99,7 +99,7 @@ class GuardLine {
       return { success: true };
     }
 
-    let offset = (this.aard.canvas.width * this.settings.active.arDetect.guardLine.ignoreEdgeMargin) << 2;
+    let offset = (this.aard.glCanvas.width * this.settings.active.arDetect.guardLine.ignoreEdgeMargin) << 2;
 
     let offenders = [];
     let offenderCount = -1; // doing it this way means first offender has offenderCount==0. Ez index.
@@ -117,8 +117,8 @@ class GuardLine {
 
     // <<<=======| checking upper row |========>>>
 
-    rowStart = ((edge_upper * this.aard.canvas.width) << 2) + offset;
+    rowStart = ((edge_upper * this.aard.glCanvas.width) << 2) + offset;
-    rowEnd = rowStart + ( this.aard.canvas.width << 2 ) - (offset * 2);
+    rowEnd = rowStart + ( this.aard.glCanvas.width << 2 ) - (offset * 2);
 
     if (Debug.debugCanvas.enabled && Debug.debugCanvas.guardLine) {
       // offenderCount = this._gl_debugRowCheck(image, rowStart, rowEnd, offenders, offenderCount);
@@ -127,8 +127,8 @@ class GuardLine {
     }
     // <<<=======| checking lower row |========>>>
 
-    rowStart = ((edge_lower * this.aard.canvas.width) << 2) + offset;
+    rowStart = ((edge_lower * this.aard.glCanvas.width) << 2) + offset;
-    rowEnd = rowStart + ( this.aard.canvas.width << 2 ) - (offset * 2);
+    rowEnd = rowStart + ( this.aard.glCanvas.width << 2 ) - (offset * 2);
 
     if (Debug.debugCanvas.enabled && Debug.debugCanvas.guardLine) {
       // offenderCount = this._gl_debugRowCheck(image, rowStart, rowEnd, offenders, offenderCount);
@@ -158,7 +158,7 @@ class GuardLine {
     if(!this.imageBar.top || !this.imageBar.bottom)
       return { success: false };
 
-    let offset = (this.aard.canvas.width * this.settings.active.arDetect.guardLine.ignoreEdgeMargin) << 2;
+    let offset = (this.aard.glCanvas.width * this.settings.active.arDetect.guardLine.ignoreEdgeMargin) << 2;
 
     // TODO: implement logo check.
 
@@ -167,14 +167,14 @@ class GuardLine {
 
     // how many non-black pixels we need to consider this check a success. We only need to detect enough pixels
     // on one edge (one of the edges can be black as long as both aren't)
-    let successThreshold = (this.aard.canvas.width * this.settings.active.arDetect.guardLine.imageTestThreshold);
+    let successThreshold = (this.aard.glCanvas.width * this.settings.active.arDetect.guardLine.imageTestThreshold);
     let rowStart, rowEnd;
 
 
     // <<<=======| checking upper row |========>>>
 
-    rowStart = ((edge_upper * this.aard.canvas.width) << 2) + offset;
+    rowStart = ((edge_upper * this.aard.glCanvas.width) << 2) + offset;
-    rowEnd = rowStart + ( this.aard.canvas.width << 2 ) - (offset * 2);
+    rowEnd = rowStart + ( this.aard.glCanvas.width << 2 ) - (offset * 2);
 
     let res = false;
 
@@ -190,7 +190,7 @@ class GuardLine {
 
     // <<<=======| checking lower row |========>>>
 
-    rowStart = ((edge_lower * this.aard.canvas.width) << 2) + offset;
+    rowStart = ((edge_lower * this.aard.glCanvas.width) << 2) + offset;
     // rowEnd = rowStart + ( this.conf.canvas.width << 2 ) - (offset * 2);
 
 

src/ext/lib/ar-detect/edge-detect/EdgeDetect.ts

@@ -100,7 +100,7 @@ class EdgeDetect{
       if(guardLineOut){
         if(guardLineOut.imageFail && !guardLineOut.blackbarFail && this.conf.guardLine.blackbar.top) {
           upper_top = this.conf.guardLine.blackbar.top;
-          upper_bottom = this.conf.canvas.height >> 1;
+          upper_bottom = this.conf.glCanvas.height >> 1;
           lower_top = upper_bottom;
           lower_bottom = this.conf.guardLine.blackbar.bottom;
         } else if (! guardLineOut.imageFail && !guardLineOut.blackbarFail && this.conf.guardLine.blackbar.top) {
@@ -110,20 +110,20 @@ class EdgeDetect{
           // a logo could falsely trigger this case, so we need to add some extra margins past
           // the point marked by guardLine.blackbar. Let's say 1/8 of canvas height on either side.
           upper_top = 0;
-          upper_bottom = this.conf.guardLine.blackbar.top + (this.conf.canvas.height >> 3);
+          upper_bottom = this.conf.guardLine.blackbar.top + (this.conf.glCanvas.height >> 3);
-          lower_top = this.conf.guardLine.blackbar.bottom - (this.conf.canvas.height >> 3);
+          lower_top = this.conf.guardLine.blackbar.bottom - (this.conf.glCanvas.height >> 3);
-          lower_bottom = this.conf.canvas.height - 1;
+          lower_bottom = this.conf.glCanvas.height - 1;
         } else {
           upper_top = 0;
-          upper_bottom = (this.conf.canvas.height >> 1) /*- parseInt(this.conf.canvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
+          upper_bottom = (this.conf.glCanvas.height >> 1) /*- parseInt(this.conf.glCanvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
-          lower_top = (this.conf.canvas.height >> 1) /*+ parseInt(this.conf.canvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
+          lower_top = (this.conf.glCanvas.height >> 1) /*+ parseInt(this.conf.glCanvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
-          lower_bottom = this.conf.canvas.height - 1;
+          lower_bottom = this.conf.glCanvas.height - 1;
         }
       } else{
         upper_top = 0;
-        upper_bottom = (this.conf.canvas.height >> 1) /*- parseInt(this.conf.canvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
+        upper_bottom = (this.conf.glCanvas.height >> 1) /*- parseInt(this.conf.glCanvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
-        lower_top = (this.conf.canvas.height >> 1) /*+ parseInt(this.conf.canvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
+        lower_top = (this.conf.glCanvas.height >> 1) /*+ parseInt(this.conf.glCanvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
-        lower_bottom = this.conf.canvas.height - 1;
+        lower_bottom = this.conf.glCanvas.height - 1;
       }
 
       this.logger.log('info', 'arDetect', '[EdgeDetect::findCandidates] searching for candidates on ranges', upper_top, '<->', upper_bottom, ';', lower_top, '<->', lower_bottom);
@@ -157,7 +157,7 @@ class EdgeDetect{
     // NOTE: this is very simple and will need to be reworked in case we ever
     //       go for quorum-based edge detection. (Probably not gonna happen)
     const topPoint = {
-      row:  this.conf.canvas.height,
+      row:  this.conf.glCanvas.height,
       gradient: false,        // does current row have a gradient sample
       noGradient: false,      // does current row have 100% confirmed edge sample
     }
@@ -220,7 +220,7 @@ class EdgeDetect{
       } else {
         // if top gradient-only sample is closer to the edge than the bottom sample,
         // validation also fails. Otherwise, we can assume success.
-        return (topPoint.row >= this.conf.canvas.height - bottomPoint.row);
+        return (topPoint.row >= this.conf.glCanvas.height - bottomPoint.row);
       }
     }
 
@@ -229,7 +229,7 @@ class EdgeDetect{
     // whether gradient-only result of bottom row is closer to the edge than than the top
     // sample.
     if (! bottomPoint.noGradient) {
-      return (topPoint.row < this.conf.canvas.height - bottomPoint.row);
+      return (topPoint.row < this.conf.glCanvas.height - bottomPoint.row);
     }
 
     return false;
@@ -240,8 +240,8 @@ class EdgeDetect{
     let edgeCandidatesBottom = {count: 0};
 
     let detections;
-    let canvasWidth = this.conf.canvas.width;
+    let canvasWidth = this.conf.glCanvas.width;
-    let canvasHeight = this.conf.canvas.height;
+    let canvasHeight = this.conf.glCanvas.height;
 
     let sampleStart, sampleEnd, loopEnd;
     let sampleRow_black, sampleRow_color;
@@ -352,7 +352,7 @@ class EdgeDetect{
   edgePostprocess(edges){
     let edgesTop = [];
     let edgesBottom = [];
-    let alignMargin = this.conf.canvas.height * this.settings.active.arDetect.allowedMisaligned;
+    let alignMargin = this.conf.glCanvas.height * this.settings.active.arDetect.allowedMisaligned;
 
     let missingEdge = edges.edgeCandidatesTopCount == 0 || edges.edgeCandidatesBottomCount == 0;
 
@@ -377,7 +377,7 @@ class EdgeDetect{
       for(let e in edges.edgeCandidatesBottom){
         let edge = edges.edgeCandidatesBottom[e];
         edgesBottom.push({
-          distance: this.conf.canvas.height - edge.offset,
+          distance: this.conf.glCanvas.height - edge.offset,
           absolute: edge.offset,
           count: edge.count
         });
@@ -542,11 +542,11 @@ class EdgeDetect{
     blackbarThreshold = this.conf.blackLevel + this.settings.active.arDetect.blackbar.threshold;
 
 
-    let middleRowStart = (this.conf.canvas.height >> 1) * this.conf.canvas.width;
+    let middleRowStart = (this.conf.glCanvas.height >> 1) * this.conf.glCanvas.width;
-    let middleRowEnd = middleRowStart + this.conf.canvas.width - 1;
+    let middleRowEnd = middleRowStart + this.conf.glCanvas.width - 1;
 
     let rowStart = middleRowStart << 2;
-    let midpoint = (middleRowStart + (this.conf.canvas.width >> 1)) << 2
+    let midpoint = (middleRowStart + (this.conf.glCanvas.width >> 1)) << 2
     let rowEnd = middleRowEnd << 2;
 
     let edge_left = -1, edge_right = -1;
@@ -564,7 +564,7 @@ class EdgeDetect{
     // check on the right
     for(let i = rowEnd; i > midpoint; i-= 4){
       if(image[i] > blackbarThreshold || image[i+1] > blackbarThreshold || image[i+2] > blackbarThreshold){
-        edge_right =  this.conf.canvas.width - ((i - rowStart) >> 2);
+        edge_right =  this.conf.glCanvas.width - ((i - rowStart) >> 2);
         break;
       }
     }