Additional AardGL fixes

.nvmrc

@@ -0,0 +1 @@
+14.4.0

.vscode/launch.json

@@ -8,10 +8,16 @@
             "name": "Launch addon",
             "type": "firefox",
             "request": "launch",
-            "port":6000,
+            "port": 6000,
             "reAttach": true,
             "addonType": "webExtension",
             "addonPath": "${workspaceFolder}/dist-ff",
+            "pathMappings": [
+                {
+                    "url": "webpack:///",
+                    "path": "${workspaceFolder}/src/"
+                }
+            ]
         }
     ],
     "firefox": {

src/ext/lib/Logger.js

@@ -330,7 +330,7 @@ class Logger {
   }
 
   canLogFile(component) {
-    if (!this.conf.fileOptions.enabled || this.temp_disable) {
+    if (!(this.conf.fileOptions?.enabled) || this.temp_disable) {
       return false;
     }
     if (Array.isArray(component) && component.length ) {

src/ext/lib/ar-detect/AardGl.js

@@ -1,4 +1,3 @@
-
 import Debug from '../../conf/Debug';
 import EdgeDetect from './edge-detect/EdgeDetect';
 import EdgeStatus from './edge-detect/enums/EdgeStatusEnum';
@@ -44,6 +43,16 @@ class AardGl {
 
     this.canDoFallbackMode = false; 
     this.logger.log('info', 'init', `[AardGl::ctor] creating new AardGl. arid: ${this.arid}`);
+
+    this.glData = {
+      positionBuffer: null,
+      textureCoordsBuffer: null,
+      textureCoordsLocation: null
+    };
+
+    // delete this:
+    this.count = 0;
+    this.greenC = true;
   }
 
   /**
@@ -198,11 +207,34 @@ class AardGl {
   }
   //#endregion
   //#region WebGL helpers
+  glInitBuffers(width, height) {
+    // create buffers and bind them
+    this.glData.positionBuffer = this.gl.createBuffer();
+    this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.glData.positionBuffer);
+
+    // create rectangle for drawing
+    this.glSetRectangle(this.gl, width, height);
+
+    // create texture coordinate buffer
+    this.glData.textureCoordsBuffer = this.gl.createBuffer();
+    this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.glData.textureCoordsBuffer);
+
+    // create index buffer
+    this.glData.indexBuffer = this.gl.createBuffer();
+    this.gl.bindBuffer(this.gl.ELEMENT_ARRAY_BUFFER, this.glData.indexBuffer);
+    
+    // This array defines each face as two triangles, using the
+    // indices into the vertex array to specify each triangle's
+    // position.
+    const indices = [0, 1, 2, 3, 4, 5];
+    this.gl.bufferData(this.gl.ELEMENT_ARRAY_BUFFER, new Uint16Array(indices), this.gl.STATIC_DRAW);
+  }
+
   glSetRectangle(glContext, width, height) {
     glContext.bufferData(glContext.ARRAY_BUFFER, new Float32Array([
-      0, 0,
-      width, 0,
-      0, height,
+      0, 0,       //
+      width, 0,   // this line are swapped over for experiment
+      0, height,  // this triangle is flipped. This and
       0, height,
       width, 0,
       width, height
@@ -224,8 +256,8 @@ class AardGl {
 
     // check if shader was compiled successfully
     if (! glContext.getShaderParameter(shader, this.gl.COMPILE_STATUS)) {
+      this.logger.log('error', ['init', 'debug', 'arDetect'], `%c[AardGl::setupShader] <@${this.arid}> Failed to setup shader. Error given:`, _ard_console_stop, this.gl.getShaderInfoLog(shader));
       glContext.deleteShader(shader);
-      this.logger.log('error', ['init', 'debug', 'arDetect'], `%c[AardGl::setupShader] <@${this.arid}> Failed to setup shader.`, _ard_console_stop);
       return null;
     }
 
@@ -241,16 +273,25 @@ class AardGl {
     console.log(glContext, shaders);
     const program = glContext.createProgram();
     for (const shader of shaders) {
+      console.log("shader", shader);
       glContext.attachShader(program, shader);
     }
     glContext.linkProgram(program);
     if (! glContext.getProgramParameter(program, glContext.LINK_STATUS)) {
-      glContext.deleteShader(shader);
-      this.logger.log('error', ['init', 'debug', 'arDetect'], `%c[AardGl::setupProgram] <@${this.arid}> Failed to setup program.`, _ard_console_stop);
+      this.logger.log('error', ['init', 'debug', 'arDetect'], `%c[AardGl::setupProgram] <@${this.arid}> Failed to setup program.`, glContext.getProgramInfoLog(program), _ard_console_stop);
       return null;
     }
 
-    return program;
+    return {
+      program,
+      attribLocations: {
+        vertexPosition: this.gl.getAttribLocation(program, 'aVertexPosition'),
+        textureCoord: this.gl.getAttribLocation(program, 'aTextureCoord'),
+      },
+      uniformLocations: {
+        u_frame: this.gl.getUniformLocation(program, 'u_frame'),
+      }
+    };
   }
   //#endregion
 
@@ -271,7 +312,7 @@ class AardGl {
         throw "Settings prevent autoar from starting"
       }
     } catch (e) {
-      this.logger.log('error', 'init', `%c[AardGl::init] <@${this.arid}> Initialization failed.`, _ard_console_stop, e);
+      this.logger.log('error', ['init', 'debug', 'aard'], `%c[AardGl::init] <@${this.arid}> Initialization failed.`, _ard_console_stop, e);
     }
   }
 
@@ -326,6 +367,17 @@ class AardGl {
     this.blackframeCanvas.width = this.settings.active.aard.canvasDimensions.blackframeCanvas.width;
     this.blackframeCanvas.height = this.settings.active.aard.canvasDimensions.blackframeCanvas.height;
 
+    // FOR DEBUG PURPOSES ONLY — REMOVE!
+    var body = document.getElementsByTagName('body')[0];
+
+    this.canvas.style.position = "fixed";
+    this.canvas.style.left = `50px`;
+    this.canvas.style.top = `64px`;
+    this.canvas.style.zIndex = 10002;
+
+    body.appendChild(this.canvas);
+    // END FOR DEBUG PURPOSES ONLY
+
     // this.context = this.canvas.getContext("2d");
 
     this.pixelBuffer = new Uint8Array(cwidth * cheight * 4);
@@ -333,8 +385,67 @@ class AardGl {
     //
     // [2] SETUP WEBGL STUFF —————————————————————————————————————————————————————————————————————————————————
     //#region webgl setup
+    
+    this.glSetup(cwidth, cheight);
+
+    console.log("glsetup complete")
+
+    // do setup once
+    // tho we could do it for every frame
+    this.canvasScaleFactor = cheight / this.video.videoHeight;
+
+    //#endregion
+
+
+    //
+    // [5] do other things setup needs to do
+    //
+
+    // this.detectionTimeoutEventCount = 0;
+    // this.resetBlackLevel();
+
+    // // if we're restarting AardGl, we need to do this in order to force-recalculate aspect ratio
+    // this.conf.resizer.setLastAr({type: AspectRatio.Automatic, ratio: this.getDefaultAr()});
+
+    // this.canvasImageDataRowLength = cwidth << 2;
+    // this.noLetterboxCanvasReset = false;
+    
+    // if (this.settings.canStartAutoAr() ) {
+    //   this.start();
+    // }
+  
+    // if(Debug.debugCanvas.enabled){
+    //   // this.debugCanvas.init({width: cwidth, height: cheight});
+    //   // DebugCanvas.draw("test marker","test","rect", {x:5, y:5}, {width: 5, height: 5});
+    // }
+
+    this.conf.arSetupComplete = true;
+    console.log("DRAWING BUFFER SIZE:", this.gl.drawingBufferWidth, '×', this.gl.drawingBufferHeight);
+
+    // start autodetection after setup is complete
+    this.start();
+  }
+
+  glSetup(cwidth, cheight) {
     this.gl = this.canvas.getContext("webgl");
 
+    if (this.gl === null) {
+      throw new Error('Unable to initialize WebGL. WebGL may not be supported by machine.');
+    }
+
+    // set color to half-transparent blue initially, for testing purposes
+    if (process.env.CHANNEL === 'dev') {
+      try {
+        this.gl.clearColor(0, 0, 1.0, 0.5);
+        this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+      } catch (e) {
+        console.error("failing to clear channel!", e);
+      }
+    } else {
+      this.gl.clearColor(0, 0, 0.0, 0.0);
+      this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+    }
+
     // load shaders and stuff. PixelSize for horizontalAdder should be 1/sample canvas width
     const vertexShaderSrc = getBasicVertexShader();
     const horizontalAdderShaderSrc = generateHorizontalAdder(10, 1 / cwidth); // todo: unhardcode 10 as radius
@@ -344,86 +455,88 @@ class AardGl {
     const horizontalAdderShader = this.compileShader(this.gl, horizontalAdderShaderSrc, this.gl.FRAGMENT_SHADER);
 
     // link shaders to program
-    const glProgram = this.compileProgram(this.gl, [vertexShader, horizontalAdderShader]);
+    const programInfo = this.compileProgram(this.gl, [vertexShader, horizontalAdderShader]);
+    this.glProgram = programInfo.program;
+    this.glData.attribLocations = programInfo.attribLocations;
+    this.glData.uniformLocations = programInfo.uniformLocations;
 
     // look up where the vertex data needs to go
     // const positionLocation = this.gl.getAttributeLocation(glProgram, 'a_position');
     // const textureCoordsLocation = this.gl.getAttributeLocation(glProgram, 'a_textureCoords');
 
-    // create buffers and bind them
-    const positionBuffer = this.gl.createBuffer();
-    const textureCoordsBuffer = this.gl.createBuffer();
-    this.gl.bindBuffer(this.gl, positionBuffer);
-    this.gl.bindBuffer(this.gl.ARRAY_BUFFER, textureCoordsBuffer);
+    console.log("program compiled. init buffers");
+
+    this.glInitBuffers(this.settings.active.aardGl.sampleCols, cheight);
+
+    console.log("program compiled. buffer init complete");
 
-    // create a texture
     this.texture = this.gl.createTexture();
     this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture);
 
+    // texture is half-transparent blue by default. Helps with debugging.
+    // this.gl.texImage2D(
+    //   this.gl.TEXTURE_2D, // target
+    //   0,                  // level
+    //   this.gl.RGBA,       // internal format
+    //   1, 1, 0,            // width, height, border
+    //   this.gl.RGBA,       // format of content
+    //   this.gl.UNSIGNED_BYTE, // type
+    //   new ArrayBufferView([0, 0, 255, 128])
+    // );
+
     // set some parameters
-    // btw we don't need to set gl.TEXTURE_WRAP_[S|T], because it's set to repeat by default — which is what we want
-    this.gl.texParameteri(this.gl, this.gl.TEXTURE_MAG_FILTER, this.gl.NEAREST);
-    this.gl.texParameteri(this.gl, this.gl.TEXTURE_MIN_FILTER, this.gl.NEAREST);
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.NEAREST);
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.NEAREST);
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE);
+    this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE);
 
     // we need a rectangle. This is output data, not texture. This means that the size of the rectangle should be
     // [sample count] x height of the sample, as shader can sample frame at a different resolution than what gets
     // rendered here. We don't need all horizontal pixels on our output. We do need all vertical pixels, though)
     this.glSetRectangle(this.gl, this.settings.active.aard.sampleCols, cheight);
 
-    // do setup once
-    // tho we could do it for every frame
-    this.canvasScaleFactor = cheight / this.video.videoHeight;
-
-    //#endregion
-
-    //
-    // [3] detect if we're in the fallback mode and reset guardline
-    //
-
-    if (this.fallbackMode) {
-      this.logger.log('warn', 'debug', `[AardGl::setup] <@${this.arid}>  WARNING: CANVAS RESET DETECTED/we're in fallback mode - recalculating guardLine`, "background: #000; color: #ff2");
-      // blackbar, imagebar
-      this.guardLine.reset();
-    }
-
-    //
-    // [4] see if browser supports "fallback mode" by drawing a small portion of our window
-    //
-
-    try {
-      this.blackframeContext.drawWindow(window,0, 0, this.blackframeCanvas.width, this.blackframeCanvas.height, "rgba(0,0,128,1)");
-      this.canDoFallbackMode = true;
-    } catch (e) {
-      this.canDoFallbackMode = false;
-    }
-    
-    //
-    // [5] do other things setup needs to do
-    //
-
-    this.detectionTimeoutEventCount = 0;
-    this.resetBlackLevel();
-
-    // if we're restarting AardGl, we need to do this in order to force-recalculate aspect ratio
-    this.conf.resizer.setLastAr({type: AspectRatio.Automatic, ratio: this.getDefaultAr()});
-
-    this.canvasImageDataRowLength = cwidth << 2;
-    this.noLetterboxCanvasReset = false;
-    
-    if (this.settings.canStartAutoAr() ) {
-      this.start();
-    }
-  
-    if(Debug.debugCanvas.enabled){
-      // this.debugCanvas.init({width: cwidth, height: cheight});
-      // DebugCanvas.draw("test marker","test","rect", {x:5, y:5}, {width: 5, height: 5});
-    }
-
-    this.conf.arSetupComplete = true;
-    console.log("DRAWING BUFFER SIZE:", this.gl.drawingBufferWidth, '×', this.gl.drawingBufferHeight);
+    console.log("gl setup complete");
   }
 
-  drawFrame() {
+
+  drawScene() {
+    if (this.count++ % 10 === 0) {
+      this.greenC = !this.greenC;
+    }
+    // clear canvas
+    this.gl.clearColor(0, this.greenC ? 0.5 : 0, 0.75, 0.5); 
+    this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+
+    this.gl.useProgram(this.glProgram);
+
+    this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.glData.positionBuffer);
+    this.gl.vertexAttribPointer(this.glData.attribLocations.vertexPosition, 3, this.gl.FLOAT, false, 0, 0);
+
+    this.gl.bindBuffer(this.gl.ARRAY_BUFFER, this.glData.textureCoordsBuffer);
+    this.gl.vertexAttribPointer(this.glData.attribLocations.textureCoord, size, type, normalized, stride, offset)
+
+    this.gl.bindBuffer(this.gl.ELEMENT_ARRAY_BUFFER, this.glData.indexBuffer);
+
+    // run our program
+    this.gl.useProgram(this.glProgram);
+    
+    // Tell WebGL we want to affect texture unit 0
+    gl.activeTexture(gl.TEXTURE0);
+
+    // Bind the texture to texture unit 0
+    gl.bindTexture(gl.TEXTURE_2D, texture);
+
+    // Tell the shader we bound the texture to texture unit 0
+    gl.uniform1i(this.glData.uniformLocations.u_frame, 0);
+
+    // this.gl.drawElements(this.gl.TRIANGLES, 2, this.gl.UNSIGNED_BYTE, 0);
+    this.gl.drawArrays(this.gl.TRIANGLES, 0, 2)
+
+    // get the pixels back out:
+    this.gl.readPixels(0, 0, width, height, this.gl.RGBA, this.gl.UNSIGNED_BYTE, this.pixelBuffer);
+  }
+
+  updateTexture() {
     const level = 0;
     const internalFormat = this.gl.RGBA;
     const sourceFormat = this.gl.RGBA;
@@ -435,25 +548,31 @@ class AardGl {
       // TODO: check if 'width' and 'height' mean the input gets resized
       // this.gl.texImage2D(gl.TEXTURE_2D, level, internalformat, width, height, border, format, type, pixels)
     // } else {
-      this.gl.texImage2D(gl.TEXTURE_2D, level, internalformat, sourceFormat, sourceType, this.video);
+      console.log(this.video)
+      this.gl.texImage2D(this.gl.TEXTURE_2D, level, internalFormat, sourceFormat, sourceType, this.video);
     // }
 
-    // get the pixels back out:
-    this.gl.readPixels(0, 0, width, height, format, type, pixels)
+
   }
 
   async main() {
+    this.logger.log('info', 'debug', `"%c[AardGl::main] <@${this.arid}>  Entering main function.`, _ard_console_start);
     if (this._paused) {
       // unpause if paused
       this._paused = false;
       return; // main loop still keeps executing. Return is needed to avoid a million instances of autodetection
     }
+
+    console.log("we werent paused");
+
     if (!this._halted) { 
       // we are already running, don't run twice
       // this would have handled the 'paused' from before, actually.
       return;
     }
 
+    console.log("we werent halted");
+
     let exitedRetries = 10;
 
     while (!this._exited && exitedRetries --> 0) {
@@ -465,7 +584,9 @@ class AardGl {
       return;
     }
 
-    this.logger.log('info', 'debug', `%c[AardGl::main] <@${this.arid}>  Previous instance didn't exit in time. Not starting a new one.`);
+    console.log("no other instances")
+
+    this.logger.log('info', 'debug', `%c[AardGl::main] <@${this.arid}>  Starting a new instance.`);
 
     // we need to unhalt:
     this._halted = false;
@@ -478,32 +599,38 @@ class AardGl {
     let frameCheckBufferIndex = 0;
     let fcstart, fctime;
 
-    while (this && !this._halted) {
-      // NOTE: we separated tickrate and inter-check timeouts so that when video switches
-      // state from 'paused' to 'playing', we don't need to wait for the rest of the longer
-      // paused state timeout to finish.
+    this.logger.log('info', 'debug', `"%c[AardGl::start] <@${this.arid}>  Starting aardGL loop!`, _ard_console_start);
 
-      if ( (!this._manualTicks && this.canTriggerFrameCheck(lastFrameCheckStartTime)) || this._nextTick) {
-        this._nextTick = false;
+    try {
+      while (this && !this._halted) {
+        // NOTE: we separated tickrate and inter-check timeouts so that when video switches
+        // state from 'paused' to 'playing', we don't need to wait for the rest of the longer
+        // paused state timeout to finish.
 
-        lastFrameCheckStartTime = Date.now();
-        fcstart = performance.now();
-        
-        try {
-          this.frameCheck();
-        } catch (e) {
-          this.logger.log('error', 'debug', `%c[AardGl::main] <@${this.arid}>  Frame check failed:`,  "color: #000, background: #f00", e);
+        if ( (!this._manualTicks && this.canTriggerFrameCheck(lastFrameCheckStartTime)) || this._nextTick) {
+          this._nextTick = false;
+
+          lastFrameCheckStartTime = Date.now();
+          fcstart = performance.now();
+          
+          try {
+            this.frameCheck();
+          } catch (e) {
+            this.logger.log('error', 'debug', `%c[AardGl::main] <@${this.arid}>  Frame check failed:`,  "color: #000, background: #f00", e);
+          }
+
+          if (Debug.performanceMetrics) {
+            fctime = performance.now() - fcstart;
+            frameCheckTimes[frameCheckBufferIndex % frameCheckTimes.length] = fctime;
+            this.conf.pageInfo.sendPerformanceUpdate({frameCheckTimes: frameCheckTimes, lastFrameCheckTime: fctime});
+            ++frameCheckBufferIndex;
+          }
         }
 
-        if (Debug.performanceMetrics) {
-          fctime = performance.now() - fcstart;
-          frameCheckTimes[frameCheckBufferIndex % frameCheckTimes.length] = fctime;
-          this.conf.pageInfo.sendPerformanceUpdate({frameCheckTimes: frameCheckTimes, lastFrameCheckTime: fctime});
-          ++frameCheckBufferIndex;
-        }
+        await this.nextFrame();
       }
-
-      await this.nextFrame();
+    } catch (e) {
+      this.logger.log('error', 'debug', `%c[AardGl::main] <@${this.arid}>  Main autodetection loop crashed. Reason?`, e, _ard_console_stop);
     }
 
     this.logger.log('info', 'debug', `%c[AardGl::main] <@${this.arid}>  Main autodetection loop exited. Halted? ${this._halted}`,  _ard_console_stop);
@@ -511,38 +638,51 @@ class AardGl {
   }
 
   frameCheck(){
-    if(! this.video){
-      this.logger.log('error', 'debug', `%c[AardGl::frameCheck] <@${this.arid}>  Video went missing. Destroying current instance of videoData.`);
-      this.conf.destroy();
-      return;
-    }
-
-    if (!this.blackframeContext) {
-      this.init();
-    }
-    
-    var startTime = performance.now();
-
-    //
-    // [0] try drawing image to canvas
-    //
-    let imageData;
-
     try {
-      this.drawFrame();
+      if(! this.video){
+        this.logger.log('error', 'debug', `%c[AardGl::frameCheck] <@${this.arid}>  Video went missing. Destroying current instance of videoData.`);
+        this.conf.destroy();
+        return;
+      }
 
 
-      this.fallbackMode = false;
+      // we dont have blackframe canvas atm
+      // if (!this.blackframeContext) {
+      //   this.init();
+      // }
+      
+      var startTime = performance.now();
+
+      //
+      // [0] try drawing image to canvas
+      //
+      let imageData;
+
+      try {
+        // this.drawFrame();
+
+
+        this.fallbackMode = false;
+      } catch (e) {
+        this.logger.log('error', 'arDetect', `%c[AardGl::frameCheck] <@${this.arid}>  %c[AardGl::frameCheck] can't draw image on canvas. ${this.canDoFallbackMode ? 'Trying canvas.drawWindow instead' : 'Doing nothing as browser doesn\'t support fallback mode.'}`, "color:#000; backgroud:#f51;", e);
+      }
+
+      // [1] update frame
+      try {
+        this.updateTexture();
+        this.drawScene();
+      } catch (e) {
+        this.logger.log('error', 'aardGl', `%c[AardGl::frameCheck] <@${this.arid}> Something went wrong while trying to update/draw video frame with gl!`, "color:#000; backgroud:#f51;", e);
+      }
+
+      console.log("TEXTURE DRAWN!", this.pixelBuffer)
+
+      // [N] clear data
+
+      this.clearImageData(imageData);
     } catch (e) {
-      this.logger.log('error', 'arDetect', `%c[AardGl::frameCheck] <@${this.arid}>  %c[AardGl::frameCheck] can't draw image on canvas. ${this.canDoFallbackMode ? 'Trying canvas.drawWindow instead' : 'Doing nothing as browser doesn\'t support fallback mode.'}`, "color:#000; backgroud:#f51;", e);
+      this.logger.log('error', 'debug', `%c[AardGl::frameCheck] <@${this.arid}> Error during framecheck.`, "background: #000; color: #fa2", e);
     }
-
-    // [1]
-
-
-
-    
-    this.clearImageData(imageData);
   }
 
   /**

src/ext/lib/video-data/VideoData.js

@@ -179,7 +179,8 @@ class VideoData {
       return;
     }
     if(! this.arSetupComplete){
-      this.arDetector = new ArDetector(this);
+      // this.arDetector = new ArDetector(this);
+      this.arDetector = new AardGl(this);
     }
   }
 
@@ -192,7 +193,8 @@ class VideoData {
       this.arDetector.init();
     }
     else{
-      this.arDetector = new ArDetector(this);
+      // this.arDetector = new ArDetector(this);
+      this.arDetector = new AardGl(this);
       this.arDetector.init();
     }
   }