ultrawidify/src/ext/lib/ar-detect/AardGl.js

import Debug from '../../conf/Debug';
import EdgeDetect from './edge-detect/EdgeDetect';
import EdgeStatus from './edge-detect/enums/EdgeStatusEnum';
import EdgeDetectPrimaryDirection from './edge-detect/enums/EdgeDetectPrimaryDirectionEnum';
import EdgeDetectQuality from './edge-detect/enums/EdgeDetectQualityEnum';
import GuardLine from './GuardLine';
import DebugCanvas from './DebugCanvas';
import VideoAlignment from '../../../common/enums/video-alignment.enum';
import AspectRatio from '../../../common/enums/aspect-ratio.enum';
import { generateHorizontalAdder } from './gllib/shader-generators/HorizontalAdderGenerator';
import { getVertexShader } from './gllib/shaders/vertex-shader';
import { sleep } from '../Util';

/**
 * AardGl: Hardware accelerated aspect ratio detection script, based on WebGL
 */
class AardGl {

  constructor(videoData){
    this.logger = videoData.logger;
    this.conf = videoData;
    this.video = videoData.video;
    this.settings = videoData.settings;
    
    this.setupTimer = null;

    this.sampleCols = [];

    this.canFallback = true;
    this.fallbackMode = false;

    this.blackLevel = this.settings.active.arDetect.blackbar.blackLevel;

    this.arid = (Math.random()*100).toFixed();

    // ar detector starts in this state. running main() sets both to false
    this._halted = true;
    this._exited = true;

    // we can tick manually, for debugging
    this._manualTicks = false;
    this._nextTick = false;

    this.canDoFallbackMode = false; 
    this.logger.log('info', 'init', `[AardGl::ctor] creating new AardGl. arid: ${this.arid}`);
  }

  /**
   * 
   *  HELPER FUNCTIONS
   *  
   */
  //#region helpers

  canTriggerFrameCheck(lastFrameCheckStartTime) {
    if (this._paused) {
      return false;
    }
    if (this.video.ended || this.video.paused){
      // we slow down if ended or pausing. Detecting is pointless.
      // we don't stop outright in case seeking happens during pause/after video was 
      // ended and video gets into 'playing' state again
      return Date.now() - lastFrameCheckStartTime > this.settings.active.arDetect.timers.paused;
    }
    if (this.video.error){
      // če je video pavziran, še vedno skušamo zaznati razmerje stranic - ampak bolj poredko.
      // if the video is paused, we still do autodetection. We just do it less often.
      return Date.now() - lastFrameCheckStartTime > this.settings.active.arDetect.timers.error;
    }
    
    return Date.now() - lastFrameCheckStartTime > this.settings.active.arDetect.timers.playing;
  }

  isRunning(){
    return ! (this._halted || this._paused || this._exited);
  }

  scheduleInitRestart(timeout, force_reset){
    if(! timeout){
      timeout = 100;
    }
    // don't allow more than 1 instance
    if(this.setupTimer){ 
      clearTimeout(this.setupTimer);
    }
    
    var ths = this;
    this.setupTimer = setTimeout(function(){
        ths.setupTimer = null;
        try{
          ths.main();
        } catch(e) {
          this.logger('error', 'debug', `[AardGl::scheduleInitRestart] <@${this.arid}> Failed to start main(). Error:`,e);
        }
        ths = null;
      },
      timeout
    );
  }

  getTimeout(baseTimeout, startTime){
    var execTime = (performance.now() - startTime);
    
    return baseTimeout;
  }

  async nextFrame() {
    return new Promise(resolve => window.requestAnimationFrame(resolve));
  }

  getDefaultAr() {
    return this.video.videoWidth / this.video.videoHeight;
  }

  resetBlackLevel(){
    this.blackLevel = this.settings.active.arDetect.blackbar.blackLevel;    
  }

  clearImageData(id) {
    if (ArrayBuffer.transfer) {
      ArrayBuffer.transfer(id, 0);
    }
    id = undefined;
  }
  //#endregion
  //#region canvas management
  attachCanvas(canvas){
    if(this.attachedCanvas)
      this.attachedCanvas.remove();

    // todo: place canvas on top of the video instead of random location
    canvas.style.position = "absolute";
    canvas.style.left = "200px";
    canvas.style.top = "1200px";
    canvas.style.zIndex = 10000;

    document.getElementsByTagName("body")[0]
            .appendChild(canvas);
  }

  canvasReadyForDrawWindow(){
    this.logger.log('info', 'debug', `%c[AardGl::canvasReadyForDrawWindow] <@${this.arid}> canvas is ${this.canvas.height === window.innerHeight ? '' : 'NOT '}ready for drawWindow(). Canvas height: ${this.canvas.height}px; window inner height: ${window.innerHeight}px.`)

    return this.canvas.height == window.innerHeight
  }
  //#endregion
  //#region aard control

  start() {
    this.logger.log('info', 'debug', `"%c[AardGl::start] <@${this.arid}>  Starting automatic aspect ratio detection`, _ard_console_start);

    if (this.conf.resizer.lastAr.type === AspectRatio.Automatic) {
      // ensure first autodetection will run in any case
      this.conf.resizer.setLastAr({type: AspectRatio.Automatic, ratio: this.getDefaultAr()});
    }


    
    // launch main() if it's currently not running:
    this.main();
    // automatic detection starts halted. If halted=false when main first starts, extension won't run
    // this._paused is undefined the first time we run this function, which is effectively the same thing
    // as false. Still, we'll explicitly fix this here.
    this._paused = false;  
    this._halted = false;
    this._paused = false;
  }

  stop(){
    this.logger.log('info', 'debug', `"%c[AardGl::stop] <@${this.arid}>  Stopping automatic aspect ratio detection`, _ard_console_stop);
    this._halted = true;
    // this.conf.resizer.setArLastAr();
  }
  
  pause() {
    // pause only if we were running before. Don't pause if we aren't running
    // (we are running when _halted is neither true nor undefined)
    if (this._halted === false) {
      this._paused = true;
    }
  }

  unpause() {
    // pause only if we were running before. Don't pause if we aren't running
    // (we are running when _halted is neither true nor undefined)
    if (this._paused && this._halted === false) {
      this._paused = true;
    }
  }

  setManualTick(manualTick) {
    this._manualTicks = manualTick;
  }

  tick() {
    this._nextTick = true;
  }
  //#endregion
  //#region WebGL helpers
  glSetRectangle(glContext, width, height) {
    glContext.bufferData(glContext.ARRAY_BUFFER, new Float32Array([
      0, 0,
      width, 0,
      0, height,
      0, height,
      width, 0,
      width, height
    ]), glContext.STATIC_DRAW);
  }

  /**
   * Creates shader
   * @param {*} glContext    — gl context
   * @param {*} shaderSource — shader code (as returned by a shader generator, for example)
   * @param {*} shaderType   — shader type (gl[context].FRAGMENT_SHADER or gl[context].VERTEX_SHADER)
   */
  compileShader(glContext, shaderSource, shaderType) {
    const shader = glContext.createShader(shaderType);

    // load source and compile shader
    glContext.shaderSource(shader, shaderSource);
    glContext.compileShader(shader);

    // check if shader was compiled successfully
    if (! glContext.getShaderParameter(shader, gl.COMPILE_STATUS)) {
      glContext.deleteShader(shader);
      this.logger.log('error', ['init', 'debug', 'arDetect'], `%c[AardGl::setupShader] <@${this.arid}> Failed to setup shader.`, _ard_console_stop);
      return null;
    }

    return shader;
  }

  /**
   * Creates gl program
   * @param {*} glContext — gl context 
   * @param {*} shaders   — shaders (previously compiled with setupShader())
   */
  compileProgram(glContext, shaders) {
    const program = glContext.createProgram();
    for (const shader of shadersr) {
      glContext.attachShader(program, shader);
    }
    glContext.linkProgram(program);
    if (! glContext.getProgramParameter(program, glContext.LINK_STATUS)) {
      glContext.deleteShader(shader);
      this.logger.log('error', ['init', 'debug', 'arDetect'], `%c[AardGl::setupProgram] <@${this.arid}> Failed to setup program.`, _ard_console_stop);
      return null;
    }

    return program;
  }
  //#endregion

  /*
   * --------------------
   * SETUP AND CLEANUP
   * --------------------
   */

  //#region init and destroy
  init(){
    this.logger.log('info', 'init', `[AardGl::init] <@${this.arid}> Initializing autodetection.`);

    try {
      if (this.settings.canStartAutoAr()) {
        this.setup();
      } else {
        throw "Settings prevent autoar from starting"
      }
    } catch (e) {
      this.logger.log('error', 'init', `%c[AardGl::init] <@${this.arid}> Initialization failed.`, _ard_console_stop, e);
    }
  }

  destroy(){
    this.logger.log('info', 'init', `%c[AardGl::destroy] <@${this.arid}> Destroying aard.`, _ard_console_stop, e);
    // this.debugCanvas.destroy();
    this.stop();
  }
  //#endregion

  setup(cwidth, cheight){
    this.logger.log('info', 'init', `[AardGl::setup] <@${this.arid}> Starting autodetection setup.`);
    //
    // [-1] check for zero-width and zero-height videos. If we detect this, we kick the proverbial
    //      can some distance down the road. This problem will prolly fix itself soon. We'll also
    //      not do any other setup until this issue is fixed
    //
    if(this.video.videoWidth === 0 || this.video.videoHeight === 0 ){
      this.logger.log('warn', 'debug', `[AardGl::setup] <@${this.arid}> This video has zero width or zero height. Dimensions: ${this.video.videoWidth} × ${this.video.videoHeight}`);

      this.scheduleInitRestart();
      return;
    }

    //
    // [0] initiate "dependencies" first
    //

    // This is space for EdgeDetector and GuardLine init
    
    //
    // [1] initiate canvases
    //

    if (!cwidth) {
      cwidth = this.settings.active.aardGl.canvasDimensions.sampleCanvas.width;
      cheight = this.settings.active.aardGl.canvasDimensions.sampleCanvas.height;
    }

    if (this.canvas) {
      this.canvas.remove();
    }
    if (this.blackframeCanvas) {
      this.blackframeCanvas.remove();
    }

    // things to note: we'll be keeping canvas in memory only. 
    this.canvas = document.createElement("canvas");
    this.canvas.width = cwidth;
    this.canvas.height = cheight;
    this.blackframeCanvas = document.createElement("canvas");
    this.blackframeCanvas.width = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.width;
    this.blackframeCanvas.height = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.height;

    this.context = this.canvas.getContext("2d");


    //
    // [2] SETUP WEBGL STUFF —————————————————————————————————————————————————————————————————————————————————
    //#region webgl setup
    this.gl = this.canvas.getContext("webgl");

    // load shaders and stuff. PixelSize for horizontalAdder should be 1/sample canvas width
    const vertexShaderSrc = getBasicVertexShader();
    const horizontalAdderShaderSrc = generateHorizontalAdder(10, 1 / cwidth); // todo: unhardcode 10 as radius

    // compile shaders
    const vertexShader = this.compileShader(this.gl, vertexShaderSrc, this.gl.VERTEX_SHADER);
    const horizontalAdderShader = this.compileShader(this.gl, horizontalAdderShaderSrc, this.gl.FRAGMENT_SHADER);

    // link shaders to program
    const glProgram = this.compileProgram(this.gl, [vertexShader, horizontalAdderShader]);

    // look up where the vertex data needs to go
    const positionLocation = this.gl.getAttributeLocation(program, 'a_position');
    const textureCoordsLocation = this.gl.getAttributeLocation(program, 'a_textureCoords');

    // create buffers and bind them
    const positionBuffer = this.gl.createBuffer();
    const textureCoordsBuffer = this.gl.createBuffer();
    this.gl.bindBuffer(this.gl, positionBuffer);
    this.gl.bindBuffer(this.gl.ARRAY_BUFFER, textureCoordsBuffer);

    // create a texture
    const texture = this.gl.createTexture();
    this.gl.bindTexture(this.gl.TEXTURE_2D, texture);

    // set some parameters
    // btw we don't need to set gl.TEXTURE_WRAP_[S|T], because it's set to repeat by default — which is what we want
    this.gl.texParameteri(this.gl, this.gl.TEXTURE_MAG_FILTER, gl.NEAREST);
    this.gl.texParameteri(this.gl, this.gl.TEXTURE_MIN_FILTER, gl.NEAREST);

    // we need a rectangle. This is output data, not texture. This means that the size of the rectangle should be
    // [sample count] x height of the sample, as shader can sample frame at a different resolution than what gets
    // rendered here. We don't need all horizontal pixels on our output. We do need all vertical pixels, though)
    this.glSetRectangle(this.gl, this.settings.active.aard.sampleCols, cheight);

    // do setup once
    // tho we could do it for every frame
    this.canvasScaleFactor = cheight / this.video.videoHeight;

    //#endregion

    //
    // [3] detect if we're in the fallback mode and reset guardline
    //

    if (this.fallbackMode) {
      this.logger.log('warn', 'debug', `[AardGl::setup] <@${this.arid}>  WARNING: CANVAS RESET DETECTED/we're in fallback mode - recalculating guardLine`, "background: #000; color: #ff2");
      // blackbar, imagebar
      this.guardLine.reset();
    }

    //
    // [4] see if browser supports "fallback mode" by drawing a small portion of our window
    //

    try {
      this.blackframeContext.drawWindow(window,0, 0, this.blackframeCanvas.width, this.blackframeCanvas.height, "rgba(0,0,128,1)");
      this.canDoFallbackMode = true;
    } catch (e) {
      this.canDoFallbackMode = false;
    }
    
    //
    // [5] do other things setup needs to do
    //

    this.detectionTimeoutEventCount = 0;
    this.resetBlackLevel();

    // if we're restarting AardGl, we need to do this in order to force-recalculate aspect ratio
    this.conf.resizer.setLastAr({type: AspectRatio.Automatic, ratio: this.getDefaultAr()});

    this.canvasImageDataRowLength = cwidth << 2;
    this.noLetterboxCanvasReset = false;
    
    if (this.settings.canStartAutoAr() ) {
      this.start();
    }
  
    if(Debug.debugCanvas.enabled){
      // this.debugCanvas.init({width: cwidth, height: cheight});
      // DebugCanvas.draw("test marker","test","rect", {x:5, y:5}, {width: 5, height: 5});
    }

    this.conf.arSetupComplete = true;
  }

  async main() {
    if (this._paused) {
      // unpause if paused
      this._paused = false;
      return; // main loop still keeps executing. Return is needed to avoid a million instances of autodetection
    }
    if (!this._halted) { 
      // we are already running, don't run twice
      // this would have handled the 'paused' from before, actually.
      return;
    }

    let exitedRetries = 10;

    while (!this._exited && exitedRetries --> 0) {
      this.logger.log('warn', 'debug', `[AardGl::main] <@${this.arid}>  We are trying to start another instance of autodetection on current video, but the previous instance hasn't exited yet. Waiting for old instance to exit ...`);
      await sleep(this.settings.active.arDetect.timers.tickrate);
    }
    if (!this._exited) {
      this.logger.log('error', 'debug', `[AardGl::main] <@${this.arid}>  Previous instance didn't exit in time. Not starting a new one.`);
      return;
    }

    this.logger.log('info', 'debug', `%c[AardGl::main] <@${this.arid}>  Previous instance didn't exit in time. Not starting a new one.`);

    // we need to unhalt:
    this._halted = false;
    this._exited = false;

    // set initial timestamps so frame check will trigger the first time we run the loop
    let lastFrameCheckStartTime = Date.now() - (this.settings.active.aardGl.timers.playing << 1);

    const frameCheckTimes = new Array(10).fill(-1);
    let frameCheckBufferIndex = 0;
    let fcstart, fctime;

    while (this && !this._halted) {
      // NOTE: we separated tickrate and inter-check timeouts so that when video switches
      // state from 'paused' to 'playing', we don't need to wait for the rest of the longer
      // paused state timeout to finish.

      if ( (!this._manualTicks && this.canTriggerFrameCheck(lastFrameCheckStartTime)) || this._nextTick) {
        this._nextTick = false;

        lastFrameCheckStartTime = Date.now();
        fcstart = performance.now();
        
        try {
          this.frameCheck();
        } catch (e) {
          this.logger.log('error', 'debug', `%c[AardGl::main] <@${this.arid}>  Frame check failed:`,  "color: #000, background: #f00", e);
        }

        if (Debug.performanceMetrics) {
          fctime = performance.now() - fcstart;
          frameCheckTimes[frameCheckBufferIndex % frameCheckTimes.length] = fctime;
          this.conf.pageInfo.sendPerformanceUpdate({frameCheckTimes: frameCheckTimes, lastFrameCheckTime: fctime});
          ++frameCheckBufferIndex;
        }
      }

      await this.nextFrame();
    }

    this.logger.log('info', 'debug', `%c[AardGl::main] <@${this.arid}>  Main autodetection loop exited. Halted? ${this._halted}`,  _ard_console_stop);
    this._exited = true;
  }

  frameCheck(){
    if(! this.video){
      this.logger.log('error', 'debug', `%c[AardGl::frameCheck] <@${this.arid}>  Video went missing. Destroying current instance of videoData.`);
      this.conf.destroy();
      return;
    }

    if (!this.blackframeContext) {
      this.init();
    }
    
    var startTime = performance.now();

    //
    // [0] blackframe tests (they also determine whether we need fallback mode)
    //
    try {
      this.blackframeContext.drawImage(this.video, 0, 0, this.blackframeCanvas.width, this.blackframeCanvas.height);
      this.fallbackMode = false;
    } catch (e) {
      this.logger.log('error', 'arDetect', `%c[AardGl::frameCheck] <@${this.arid}>  %c[AardGl::frameCheck] can't draw image on canvas. ${this.canDoFallbackMode ? 'Trying canvas.drawWindow instead' : 'Doing nothing as browser doesn\'t support fallback mode.'}`, "color:#000; backgroud:#f51;", e);

      // nothing to see here, really, if fallback mode isn't supported by browser
      if (! this.canDoFallbackMode) {
        return;
      }
      if (! this.canvasReadyForDrawWindow()) {
        // this means canvas needs to be resized, so we'll just re-run setup with all those new parameters
        this.stop();
          
        let newCanvasWidth = window.innerHeight * (this.video.videoWidth / this.video.videoHeight);
        let newCanvasHeight = window.innerHeight;
        
        if (this.conf.resizer.videoAlignment === VideoAlignment.Center) {
          this.canvasDrawWindowHOffset = Math.round((window.innerWidth - newCanvasWidth) * 0.5);
        } else if (this.conf.resizer.videoAlignment === VideoAlignment.Left) {
          this.canvasDrawWindowHOffset = 0;
        } else {
          this.canvasDrawWindowHOffset = window.innerWidth - newCanvasWidth;
        }

        this.setup(newCanvasWidth, newCanvasHeight);
        
        return;
      } 
      // if this is the case, we'll first draw on canvas, as we'll need intermediate canvas if we want to get a
      // smaller sample for blackframe check
      this.fallbackMode = true;

      try {
        this.context.drawWindow(window, this.canvasDrawWindowHOffset, 0, this.canvas.width, this.canvas.height, "rgba(0,0,128,1)");
      } catch (e) {
        this.logger.log('error', 'arDetect', `%c[AardGl::frameCheck] can't draw image on canvas with fallback mode either. This error is prolly only temporary.`, "color:#000; backgroud:#f51;", e);
        return; // it's prolly just a fluke, so we do nothing special here
      }
      // draw blackframe sample from our main sample:
      this.blackframeContext.drawImage(this.canvas, this.blackframeCanvas.width, this.blackframeCanvas.height)

      this.logger.log('info', 'arDetect_verbose', `%c[AardGl::frameCheck] canvas.drawImage seems to have worked`, "color:#000; backgroud:#2f5;");
    }

    // [1]

    this.clearImageData(imageData);
  }

  /**
   * -------------------------
   * DATA PROCESSING HELPERS
   * -------------------------
   */
  //#region result processing
  calculateArFromEdges(edges) {
    // if we don't specify these things, they'll have some default values.
    if(edges.top === undefined){
      edges.top = 0;
      edges.bottom = 0;
      edges.left = 0;     // RESERVED FOR FUTURE — CURRENTLY UNUSED
      edges.right = 0;    // THIS FUNCTION CAN PRESENTLY ONLY HANDLE LETTERBOX
    }

    let letterbox = edges.top + edges.bottom;
    

    if (! this.fallbackMode) {
      // Since video is stretched to fit the canvas, we need to take that into account when calculating target
      // aspect ratio and correct our calculations to account for that     

      const fileAr = this.video.videoWidth / this.video.videoHeight;
      const canvasAr = this.canvas.width / this.canvas.height;
      let widthCorrected;

      if (edges.top && edges.bottom) {
        // in case of letterbox, we take canvas height as canon and assume width got stretched or squished

        if (fileAr != canvasAr) {
          widthCorrected = this.canvas.height * fileAr;
        } else {
          widthCorrected = this.canvas.width;
        }

        return widthCorrected / (this.canvas.height - letterbox);
      }
    } else {
      // fallback mode behaves a wee bit differently

      let zoomFactor = 1;
   
      // there's stuff missing from the canvas. We need to assume canvas' actual height is bigger by a factor x, where
      //   x = [video.zoomedHeight] / [video.unzoomedHeight]
      //
      // letterbox also needs to be corrected:
      //   letterbox += [video.zoomedHeight] - [video.unzoomedHeight]

      var vbr = this.video.getBoundingClientRect();
      
      zoomFactor = vbr.height / this.video.clientHeight;
      letterbox += vbr.height - this.video.clientHeight;

      var trueHeight = this.canvas.height * zoomFactor - letterbox;

      if(edges.top > 1 && edges.top <= this.settings.active.arDetect.fallbackMode.noTriggerZonePx ){
        this.logger.log('info', 'arDetect', `%c[AardGl::calculateArFromEdges] <@${this.arid}>  Edge is in the no-trigger zone. Aspect ratio change is not triggered.`)
        return;
      }
      
      // varnostno območje, ki naj ostane črno (da lahko v fallback načinu odkrijemo ožanje razmerja stranic).
      // x2, ker je safetyBorderPx definiran za eno stran.
      // safety border so we can detect aspect ratio narrowing (21:9 -> 16:9).
      // x2 because safetyBorderPx is for one side.
      trueHeight += (this.settings.active.arDetect.fallbackMode.safetyBorderPx << 1);

      return this.canvas.width * zoomFactor / trueHeight;
    }
  }

  processAr(trueAr){
    this.detectedAr = trueAr;
    
    // poglejmo, če se je razmerje stranic spremenilo
    // check if aspect ratio is changed:
    var lastAr = this.conf.resizer.getLastAr();
    if (lastAr.type === AspectRatio.Automatic && lastAr.ratio !== null){
      // spremembo lahko zavrnemo samo, če uporabljamo avtomatski način delovanja in če smo razmerje stranic
      // že nastavili.
      //
      // we can only deny aspect ratio changes if we use automatic mode and if aspect ratio was set from here.
      
      var arDiff = trueAr - lastAr.ar;
      
      if (arDiff < 0)
        arDiff = -arDiff;
      
      var arDiff_percent = arDiff / trueAr;
      
      // ali je sprememba v mejah dovoljenega? Če da -> fertik
      // is ar variance within acceptable levels? If yes -> we done
      this.logger.log('info', 'arDetect', `%c[AardGl::processAr] <@${this.arid}>  New aspect ratio varies from the old one by this much:\n`,"color: #aaf","old Ar", lastAr.ar, "current ar", trueAr, "arDiff (absolute):",arDiff,"ar diff (relative to new ar)", arDiff_percent);

      if (arDiff < trueAr * this.settings.active.arDetect.allowedArVariance){
        this.logger.log('info', 'arDetect', `%c[AardGl::processAr] <@${this.arid}>  Aspect ratio change denied — diff %: ${arDiff_percent}`, "background: #740; color: #fa2");
        return;
      }
      this.logger.log('info', 'arDetect', `%c[AardGl::processAr] <@${this.arid}>  aspect ratio change accepted — diff %: ${arDiff_percent}`, "background: #153; color: #4f9");
    }
    this.logger.log('info', 'debug', `%c[AardGl::processAr] <@${this.arid}>  Triggering aspect ratio change. New aspect ratio: ${trueAr}`, _ard_console_change);
    
    this.conf.resizer.updateAr({type: AspectRatio.Automatic, ratio: trueAr}, {type: AspectRatio.Automatic, ratio: trueAr});
  }
  //#endregion
  //#region data processing / frameCheck helpers

  //#endregion
}

var _ard_console_stop = "background: #000; color: #f41";
var _ard_console_start = "background: #000; color: #00c399";
var _ard_console_change = "background: #000; color: #ff8";

export default AardGl;