ultrawidify/src/ext/lib/aard/Aard.ts

import AspectRatioType from '@src/common/enums/AspectRatioType.enum';
import ExtensionMode from '@src/common/enums/ExtensionMode.enum';
import { ArVariant } from '@src/common/interfaces/ArInterface';
import { ExtensionEnvironment } from '@src/common/interfaces/SettingsInterface';
import EventBus from '../EventBus';
import Settings from '../settings/Settings';
import { SiteSettings } from '../settings/SiteSettings';
import VideoData from '../video-data/VideoData';
import { AardDebugUi } from './AardDebugUi';
import { AardTimer } from './AardTimers';
import { Corner } from './enums/corner.enum';
import { VideoPlaybackState } from './enums/video-playback-state.enum';
import { FallbackCanvas } from './gl/FallbackCanvas';
import { GlCanvas } from './gl/GlCanvas';
import { GlDebugCanvas, GlDebugType } from './gl/GlDebugCanvas';
import { AardCanvasStore } from './interfaces/aard-canvas-store.interface';
import { AardDetectionSample, generateSampleArray, resetSamples } from './interfaces/aard-detection-sample.interface';
import { AardStatus, initAardStatus } from './interfaces/aard-status.interface';
import { AardTestResult_SubtitleRegion, AardTestResults, initAardTestResults, resetAardTestResults, resetGuardLine, resetSubtitleScanResults } from './interfaces/aard-test-results.interface';
import { AardTimers, initAardTimers } from './interfaces/aard-timers.interface';
import { ComponentLogger } from '../logging/ComponentLogger';
import { AardPollingOptions } from './enums/aard-polling-options.enum';
import { AardSubtitleCropMode } from './enums/aard-subtitle-crop-mode.enum';
import { LetterboxOrientation } from './enums/letterbox-orientation.enum';
import { Edge } from './enums/edge.enum';
import { AardUncertainReason } from './enums/aard-letterbox-uncertain-reason.enum';
import { result } from 'lodash';


/**
 *           /\
 *          //\\         Automatic
 *         //  \\         Aspect
 *        //    \\         Ratio
 *               \\         Detector
 *      //XXXX    \\
 *     //          \\    (Totes not a Witcher reference)
 *    //            \\       (Witcher 2 best Witcher)
 *   //XXXXXXXXXXXXXX\\
 *
 */

const PIXEL_SIZE = 4;
const PIXEL_SIZE_FRACTION = 0.25;
let ROW_SIZE = -1;

export class Aard {
  //#region configuration parameters
  private logger: ComponentLogger;
  private videoData: VideoData;
  private settings: Settings;
  private siteSettings: SiteSettings;
  private eventBus: EventBus;
  private arid: string;
  private arVariant: ArVariant;

  private eventBusCommands = {
    'uw-environment-change': {
      function: (newEnvironment: ExtensionEnvironment) => {
        // console.log('received extension environment:', newEnvironment, 'player env:', this.videoData?.player?.environment);
        this.startCheck();
      }
    },
    'aard-enable-debug': {
      function: (enabled: boolean) => {
        if (enabled) {
          this.showDebugCanvas();
        } else {
          this.hideDebugCanvas();
        }
      }
    },
    'url-changed': {
      function: () => {
        this.clearAutoDisabled();
      }
    }
    //   'get-aard-timing': {
    //     function: () => this.handlePerformanceDataRequest()
      // }
  };
  //#endregion

  private video: HTMLVideoElement;

  private animationFrame: number;

  //#region internal state
  public status: AardStatus = initAardStatus();
  private timers: AardTimers = initAardTimers();
  private inFallback: boolean = false;
  private fallbackReason: any;
  private canvasStore: AardCanvasStore;
  private testResults: AardTestResults;
  private verticalTestResults: AardTestResults;
  private canvasSamples: AardDetectionSample;


  private forceFullRecheck: boolean = true;

  private debugConfig: any = {};
  private timer: AardTimer;
  private lastAnimationFrameTime: number = Infinity;
  //#endregion

  //#region getters
  get defaultAr() {
    if (!this.video) {
      return undefined;
    }

    this.video.setAttribute('crossOrigin', 'anonymous');

    const ratio = this.video.videoWidth / this.video.videoHeight;
    if (isNaN(ratio)) {
      return undefined;
    }
    return ratio;
  }

  //#endregion getters

  //#region lifecycle
  constructor(videoData: VideoData){
    this.logger = new ComponentLogger(videoData.logAggregator, 'Aard', {});
    this.videoData = videoData;
    this.video = videoData.video;
    this.settings = videoData.settings;
    this.siteSettings = videoData.siteSettings;
    this.eventBus = videoData.eventBus;

    this.eventBus.subscribeMulti(this.eventBusCommands, this);

    this.arid = (Math.random()*100).toFixed();

    // we can tick manually, for debugging
    this.logger.log('ctor', `creating new ArDetector. arid: ${this.arid}`);

    this.timer = new AardTimer();
    this.init();
  }

  /**
   * Initializes Aard with default values and starts autodetection loop.
   * This method should only ever be called from constructor.
   */
  private init() {
    this.canvasStore = {
      main: this.createCanvas('main-gl')
    };


    this.canvasSamples = {
      top: generateSampleArray(
        this.settings.active.arDetect.sampling.staticCols,
        this.settings.active.arDetect.canvasDimensions.sampleCanvas.width
      ),
      bottom: generateSampleArray(
        this.settings.active.arDetect.sampling.staticCols,
        this.settings.active.arDetect.canvasDimensions.sampleCanvas.width
      ),
      left: generateSampleArray(
        this.settings.active.arDetect.sampling.staticCols,
        this.settings.active.arDetect.canvasDimensions.sampleCanvas.height
      ),
      right: generateSampleArray(
        this.settings.active.arDetect.sampling.staticCols,
        this.settings.active.arDetect.canvasDimensions.sampleCanvas.height
      )
    };


    // try {
    //   this.showDebugCanvas();
    // } catch (e) {
    //   console.error('FALIED TO CREATE DEBUGG CANVAS', e);
    // }

    try {
      if (this.settings.active.ui.dev?.aardDebugOverlay?.showOnStartup) {
        this.showDebugCanvas();
      }
    } catch (e) {
      console.error(`[uw::aard] failed to create debug UI:`, e);
    }

    this.startCheck();
  }

  private createCanvas(canvasId: string, canvasType?: 'webgl' | 'legacy') {
    ROW_SIZE = this.settings.active.arDetect.canvasDimensions.sampleCanvas.width * PIXEL_SIZE;

    if (canvasType) {
      if (canvasType === this.settings.active.arDetect.aardType || this.settings.active.arDetect.aardType === 'auto') {
        if (canvasType === 'webgl') {
          return new GlCanvas({...this.settings.active.arDetect.canvasDimensions.sampleCanvas, id: 'main-gl'});
        } else if (canvasType === 'legacy') {
          return new FallbackCanvas({...this.settings.active.arDetect.canvasDimensions.sampleCanvas, id: 'main-legacy'});
        } else {
          // TODO: throw error
        }
      } else {
        // TODO: throw error
      }

    }

    if (['auto', 'webgl'].includes(this.settings.active.arDetect.aardType)) {
      try {
        return new GlCanvas({...this.settings.active.arDetect.canvasDimensions.sampleCanvas, id: 'main-gl'});
      } catch (e) {
        if (this.settings.active.arDetect.aardType !== 'webgl') {
          return new FallbackCanvas({...this.settings.active.arDetect.canvasDimensions.sampleCanvas, id: 'main-legacy'});
        }
        this.logger.error('createCanvas', 'could not create webgl canvas:', e);
        this.eventBus.send('uw-config-broadcast', {type: 'aard-error', aardErrors: {webglError: true}});
        throw e;
      }
    } else if (this.settings.active.arDetect.aardType === 'legacy') {
      return new FallbackCanvas({...this.settings.active.arDetect.canvasDimensions.sampleCanvas, id: 'main-legacy'});
    } else {
      this.logger.error('createCanvas', 'invalid value in settings.arDetect.aardType:', this.settings.active.arDetect.aardType);
      this.eventBus.send('uw-config-broadcast', {type: 'aard-error', aardErrors: {invalidSettings: true}});
      throw 'AARD_INVALID_SETTINGS';
    }
  }

  /**
   * Creates and shows debug canvas
   * @param canvasId
   */
  private showDebugCanvas() {
    if (!this.canvasStore.debug) {
      this.canvasStore.debug = new GlDebugCanvas({...this.settings.active.arDetect.canvasDimensions.sampleCanvas, id: 'uw-debug-gl'});
    }
    this.canvasStore.debug.enableFx();
    if (!this.debugConfig.debugUi) {
      this.debugConfig.debugUi = new AardDebugUi(this);
      this.debugConfig.debugUi.initContainer();
      this.debugConfig.debugUi.attachCanvases(this.canvasStore.main.canvas, this.canvasStore.debug.canvas);

      // if we don't draw a dummy frame from _real_ sources, we can't update buffer later
      this.canvasStore.debug.drawVideoFrame(this.canvasStore.main.canvas);
    }
  }

  private hideDebugCanvas() {
    if (this.debugConfig.debugUi) {
      this.debugConfig?.debugUi.destroyContainer();
      this.debugConfig.debugUi = undefined;
    }
  }
  //#endregion

  /**
   * Checks whether autodetection can run
   */
  startCheck(arVariant?: ArVariant) {
    this.arVariant = arVariant;

    if (!this.videoData.player) {
      // console.warn('Player not detected!');
      // console.log('--- video data: ---\n', this.videoData);
      return;
    }
    if (this.siteSettings.data.enableAard[this.videoData.player.environment] === ExtensionMode.Enabled) {
      this.start();
    } else {
      this.stop();
    }
  }

  /**
   * Clears autoDisable flag
   */
  clearAutoDisabled() {
    this.status.autoDisabled = false;
    this.timers.autoDisableAt = undefined;
  }

  /**
   * Starts autodetection loop.
   */
  start() {
    this.clearAutoDisabled();
    this.forceFullRecheck = true;
    if (this.videoData.resizer.lastAr.type === AspectRatioType.AutomaticUpdate) {
      // ensure first autodetection will run in any case
      this.videoData.resizer.lastAr = {type: AspectRatioType.AutomaticUpdate, ratio: this.defaultAr};
    }

    // do full reset of test samples
    this.testResults = initAardTestResults(this.settings.active.arDetect);
    this.verticalTestResults = initAardTestResults(this.settings.active.arDetect);

    if (this.animationFrame) {
      window.cancelAnimationFrame(this.animationFrame);
    }

    this.status.aardActive = true;
    this.animationFrame = window.requestAnimationFrame( (ts: DOMHighResTimeStamp) => this.onAnimationFrame(ts));

    // set auto-disable timer if detection timeout is set
    if (this.settings.active.arDetect.autoDisable.ifNotChanged) {
      this.timers.autoDisableAt = Date.now() + this.settings.active.arDetect.autoDisable.ifNotChangedTimeout;
    }
  }

  /**
   * Runs autodetection ONCE.
   * If autodetection loop is running, this will also stop autodetection loop.
   */
  step(options?: {noCache?: boolean}) {
    this.stop();

    if (options?.noCache) {
      this.testResults = initAardTestResults(this.settings.active.arDetect);
      this.verticalTestResults = initAardTestResults(this.settings.active.arDetect);
    }

    this.main();
  }

  /**
   * Stops autodetection.
   */
  stop() {
    if (this.animationFrame) {
      window.cancelAnimationFrame(this.animationFrame);
    }
  }

  //#region animationFrame, scheduling, and other shit
  /**
   * Checks whether conditions for granting a frame check are fulfilled
   * @returns
   */
  private canTriggerFrameCheck() {

    // console.log('ard check status:', this.status);

    // if video was paused & we know that we already checked that frame,
    // we will not check it again.
    const videoState = this.getVideoPlaybackState();
    const polling = this.settings.active.arDetect.polling;
    const now = Date.now();

    if (videoState !== VideoPlaybackState.Playing) {
      if (this.status.lastVideoStatus === videoState) {
        return false;
      }
    }
    if (this.status.autoDisabled) {
      return false;
    }
    if (this.timers.autoDisableAt < now) {
      this.status.autoDisabled = true;
      return false;
    }
    this.status.lastVideoStatus = videoState;

    const tabVisible = document.visibilityState === 'visible';
    if (!tabVisible && polling.runInBackgroundTabs === AardPollingOptions.No) {
      return false;
    }
    if (this.videoData.player.isTooSmall && polling.runOnSmallVideos === AardPollingOptions.No) {
      return false;
    }

    const isActive = (tabVisible || polling.runInBackgroundTabs !== AardPollingOptions.Reduced)
       && (!this.videoData.player.isTooSmall || polling.runOnSmallVideos !== AardPollingOptions.Reduced);
    const nextCheck = isActive ? this.timers.nextFrameCheckTime : this.timers.reducedPollingNextCheckTime;

    if (now < nextCheck) {
      return false;
    }

    this.timers.nextFrameCheckTime = now + this.settings.active.arDetect.timers.playing;
    this.timers.reducedPollingNextCheckTime = now + this.settings.active.arDetect.timers.playingReduced;
    return true;
  }

  /**
   * Bootstraps the main loop.
   *
   * Honestly this doesn't need description, but I want to put some green
   * between two adjacent functions.
   */
  private onAnimationFrame(ts: DOMHighResTimeStamp) {
    if (this.canTriggerFrameCheck()) {
      resetAardTestResults(this.testResults);
      resetSamples(this.canvasSamples);
      resetSubtitleScanResults(this.testResults);
      this.main();
      this.forceFullRecheck = false;
    } else {
    }
    this.animationFrame = window.requestAnimationFrame( (ts: DOMHighResTimeStamp) => this.onAnimationFrame(ts));
  }
  //#endregion

  /**
   * Main loop for scanning aspect ratio changes
   */
  private async main() {
    const arConf =  this.settings.active.arDetect;

    try {
      this.timer.next();

      let imageData: Uint8Array;
      this.timer.current.start = performance.now();

      // We abuse a do-while loop to eat our cake (get early returns)
      // and have it, too (if we return early, we still execute code
      // at the end of this function)
      scanFrame:
      {
        imageData = await new Promise<Uint8Array>(
          resolve => {
            try {
              this.canvasStore.main.drawVideoFrame(this.video);
              this.timer.current.draw = performance.now() - this.timer.current.start;
              resolve(this.canvasStore.main.getImageData());
            } catch (e) {
              if (e.name === 'SecurityError') {
                this.eventBus.send('uw-config-broadcast', {type: 'aard-error', aardErrors: {cors: true}});
                this.stop();
              }
              if (this.canvasStore.main instanceof FallbackCanvas) {
                if (this.inFallback) {
                  this.eventBus.send('uw-config-broadcast', {type: 'aard-error', aardErrors: this.fallbackReason});
                  this.stop();
                } else {
                  this.eventBus.send('uw-config-broadcast', {type: 'aard-error', aardErrors: {fallbackCanvasError: true}});
                  this.stop();
                }
              } else {
                if (arConf.aardType === 'auto') {
                  this.canvasStore.main.destroy();
                  this.canvasStore.main = this.createCanvas('main-gl', 'legacy');
                }
                this.inFallback = true;
                this.fallbackReason = {cors: true};

                if (arConf.aardType !== 'auto') {
                  this.stop();
                }
              }
            }
          }
        );
        this.timer.current.getImage = performance.now() - this.timer.current.start;

        // STEP 1:
        // Test if corners are black. If they're not, we can immediately quit the loop.
        // For performances of measurements, checking orientation of letterbox is part of fastBlackLevel
        const lastValidLetterboxOrientation = this.testResults.lastValidLetterboxOrientation;

        orientationCheck:
        {
          this.getBlackLevelFast(
            imageData, 3, 1,
            arConf.canvasDimensions.sampleCanvas.width,
            arConf.canvasDimensions.sampleCanvas.height
          );

          if (this.testResults.letterboxOrientation === LetterboxOrientation.NotLetterbox) {
            break orientationCheck;
          }

          this.letterboxOrientationScan(
            imageData,
            arConf.canvasDimensions.sampleCanvas.width,
            arConf.canvasDimensions.sampleCanvas.height
          );
        }


        this.timer.current.fastBlackLevel = performance.now() - this.timer.current.start;

        // if we detect no letterbox, we don't test anything — instead, we immediately reset
        if (this.testResults.letterboxOrientation === LetterboxOrientation.NotLetterbox) {
          // TODO: reset aspect ratio to "AR not applied"
          this.testResults.lastStage = 1;
          this.testResults.letterboxSize = 0;
          this.testResults.letterboxOffset = 0;
          resetGuardLine(this.testResults);

          break scanFrame;
        }

        // If we detect both letterbox and pillarbox, we keep things as they are but avoid scanning further
        if (this.testResults.letterboxOrientation === LetterboxOrientation.Both) {
          this.testResults.lastStage = 1;

          break scanFrame;
        }

        // If lastValidLetterboxOrientation changed, we reset guard line (& gang), but continue processing
        if (lastValidLetterboxOrientation !== this.testResults.lastValidLetterboxOrientation) {
          this.forceFullRecheck = true;
        }

        // We only do subtitle check when orientation is letterbox
        if (this.testResults.letterboxOrientation === LetterboxOrientation.Letterbox) {
          this.subtitleScan(
            imageData,
            arConf.canvasDimensions.sampleCanvas.width,
            arConf.canvasDimensions.sampleCanvas.height,
            false,
          );
        }

      }

      // Note that subtitle check should reset aspect ratio outright, regardless of what other tests revealed.
      // Also note that subtitle check should run on newest aspect ratio data, rather than lag one frame behind
      // But implementation details are something for future Tam to figure out

      // If forceFullRecheck is set, then 'not letterbox' should always force-reset the aspect ratio
      // (as aspect ratio may have been set manually while autodetection was off)

      // If debugging is enable,
      this.canvasStore.debug?.drawBuffer(imageData);

      processUpdate:
      {
        if (this.testResults.letterboxOrientation === LetterboxOrientation.NotLetterbox) {
          // console.warn('DETECTED NOT LETTERBOX! (resetting)')
          this.timer.arChanged();
          this.updateAspectRatio(this.defaultAr, {forceReset: true});
          break processUpdate;
        }

        if (this.testResults.subtitleDetected && arConf.subtitles.subtitleCropMode !== AardSubtitleCropMode.CropSubtitles) {
          if (arConf.subtitles.subtitleCropMode === AardSubtitleCropMode.ResetAR) {
            this.updateAspectRatio(this.defaultAr, {forceReset: true});
            this.timers.pauseUntil = Date.now() + arConf.subtitles.resumeAfter;

          } else if (arConf.subtitles.subtitleCropMode === AardSubtitleCropMode.ResetAndDisable) {
            this.updateAspectRatio(this.defaultAr, {forceReset: true});
            this.status.autoDisabled = true;
          }

          break processUpdate;
        }

        // if detection is uncertain, we don't do anything at all (unless if guardline was broken, in which case we reset)
        if (this.testResults.aspectRatioUncertain) {
          // this.timer.arChanged();

          // if (
          //   this.testResults.aspectRatioCheck.frontCandidate < this.testResults.guardLine.front
          //   || this.testResults.aspectRatioCheck.backCandidate > this.testResults.guardLine.back
          //   || this.testResults.aspectRatioCheck.frontCandidate === -1
          //   || this.testResults.aspectRatioCheck.backCandidate === -1
          // ) {
          //   this.updateAspectRatio(this.defaultAr, {uncertainDetection: true, forceReset: true});
          // }

          break processUpdate;
        }

        // TODO: emit debug values if debugging is enabled
        this.testResults.isFinished = true;

        this.testResults.guardLine.front = this.testResults.aspectRatioCheck.frontCandidate;
        this.testResults.guardLine.back = this.testResults.aspectRatioCheck.backCandidate;


        const finalAr = this.getAr();
        if (finalAr > 0) {
          this.updateAspectRatio(finalAr);
        } else {
          this.testResults.aspectRatioInvalid = true;
          this.testResults.aspectRatioInvalidReason = finalAr.toFixed(3);
        }
        // }

        // if we got "no letterbox" OR aspectRatioUpdated
      }

      if (this.canvasStore.debug) {
        // this.canvasStore.debug.drawBuffer(imageData);
        this.timer.getAverage();
        this.debugConfig?.debugUi?.updateTestResults(this.testResults, this.timers);
      }
    } catch (e) {
      console.warn('[Ultrawidify] Aspect ratio autodetection crashed for some reason.\n\nsome reason:', e);
      this.videoData.resizer.setAr({type: AspectRatioType.AutomaticUpdate, ratio: this.defaultAr, variant: this.arVariant});
    }
  }

  private getVideoPlaybackState(): VideoPlaybackState {
    try {
      if (this.video.ended) {
        return VideoPlaybackState.Ended;
      } else if (this.video.paused) {
        return VideoPlaybackState.Paused;
      } else if (this.video.error) {
        return VideoPlaybackState.Error;
      } else {
        return VideoPlaybackState.Playing;
      }
    } catch (e) {
      this.logger.warn('getVideoPlaybackState]', `There was an error while determining video playback state.`, e);
      return VideoPlaybackState.Error;
    }
  }


  //#region buffer tests
  /**
   * Get black level of a given frame. We sample black level on very few
   * positions — just the corners of the frame. If letterboxing or pillarboxing
   * exists, then pixels in the corners of the frame should be the blackest
   * it gets.
   *
   * Sampling pattern are four lines, each shooting from its respective corner.
   * Value of 'sample' parameter determines how many pixels along this line we
   * are going to sample. Offset means how many pixels of those four lines we
   * are going to skip before we start sampling.
   *
   *    x→ 0 1 ...                 ... x-1
   *  y↓ × ------------... ...------------ ×
   *   0 | 1                             1 |
   *   1 |   2                         2   |
   *   : |     .                     .     :
   *     :       .                 .
   *
   *     :       .                 .       :
   *     |     .                     .     |
   *     |   2                         2   |
   * h-1 | 1                             1 |
   *     × ------------... ...------------ ×
   *
   *
   *                              IMPORTANT NOTES
   *  <> imageData is one-dimensional array, so we need to account for that.
   *  <> blackLevel is the darkest brightest subpixel detected
   *  <> If image has no crop, then this function WILL NOT get the true black level.
   *     In that case, we don't get an accurate black level, but we know straight
   *     away that the image is uncropped. If image is uncropped, we can skip other,
   *     more expensive tests.
   *
   * @param imageData array of pixels (4 bytes/fields per pixel)
   * @param samples number of samples per corner
   * @param width width of the frame
   * @param height height of the frame
   */
  private getBlackLevelFast(imageData: Uint8Array, samples: number, offset: number, width: number, height: number) {
    // there's 4 points for each sample, and 3 components for each of the sampling points.
    const pixelValues = new Array<number>(samples * 12);
    let pvi = 0;

    /**
     * We should ensure we are accessing pixels in ordered manner in order to
     * take advantage of data locality.
     */
    const end = offset + samples;
    for (let i = offset; i < end; i++) {
      const px_r = (i * ROW_SIZE) + (i * PIXEL_SIZE);    // red component starts here
      pixelValues[pvi++] = imageData[px_r];
      pixelValues[pvi++] = imageData[px_r + 1];
      pixelValues[pvi++] = imageData[px_r + 2];
      imageData[px_r + 3] = GlDebugType.BlackLevelSample;

      const endpx_r = px_r + ROW_SIZE - (i * PIXEL_SIZE * 2) - PIXEL_SIZE;  // - twice the offset to mirror the diagonal
      pixelValues[pvi++] = imageData[endpx_r];
      pixelValues[pvi++] = imageData[endpx_r + 1];
      pixelValues[pvi++] = imageData[endpx_r + 2];
      imageData[endpx_r + 3] = GlDebugType.BlackLevelSample;
    }

    // now let's populate the bottom two corners
    for (let i = end; i --> offset;) {
      const row = height - i - 1;  // since first row is 0, last row is height - 1

      const px_r = (row * ROW_SIZE) + (i * PIXEL_SIZE);
      pixelValues[pvi++] = imageData[px_r];
      pixelValues[pvi++] = imageData[px_r + 1];
      pixelValues[pvi++] = imageData[px_r + 2];
      imageData[px_r + 3] = GlDebugType.BlackLevelSample;

      const endpx_r = px_r + (ROW_SIZE) - (i * PIXEL_SIZE * 2) - PIXEL_SIZE;  // - twice the offset to mirror the diagonal
      pixelValues[pvi++] = imageData[endpx_r];
      pixelValues[pvi++] = imageData[endpx_r + 1];
      pixelValues[pvi++] = imageData[endpx_r + 2];
      imageData[endpx_r + 3] = GlDebugType.BlackLevelSample;
    }

    let min = 255;
    let avg = 0;
    let p = 0;

    for (let i = 0; i < pixelValues.length; i++) {
      p = pixelValues[i];
      i++;

      if (p < pixelValues[i]) {
        p = pixelValues[i];
      }
      i++;

      if (p < pixelValues[i]) {
        p = pixelValues[i];
      }

      avg += p;
      if (p < min) {
        min = p;
      }
    }

    // While there's 4 bytes / 3 values per pixel, a
    // avg only contains highest subpixel ... so we really
    // only take one sample per pixel instead of 3/4
    avg = avg / samples;

    if (avg > this.testResults.blackThreshold) {
      this.testResults.letterboxOrientation = LetterboxOrientation.NotLetterbox;
    }

    // only update black level if not letterbox.
    // NOTE: but maybe we could, if blackLevel can only get lower than
    // the default value.
    if (this.testResults.letterboxOrientation === LetterboxOrientation.NotLetterbox) {
      this.testResults.aspectRatioUncertain = false;
    }

    if (min < this.testResults.blackLevel) {
      this.testResults.blackLevel = min;
      this.testResults.blackThreshold = min + 16;
    }
  }

  /**
   * Checks orientation of black bars.
   * @param imageData
   * @param width
   * @param height
   */
  private letterboxOrientationScan(imageData: Uint8Array, width: number, height: number) {
    const lastPixelOffset = ROW_SIZE - PIXEL_SIZE;
    const imageSize = ROW_SIZE * height;

    const xLimit = this.settings.active.arDetect.letterboxOrientationScan.letterboxLimit;
    const yLimit = this.settings.active.arDetect.letterboxOrientationScan.pillarboxLimit;

    let letterbox = true, pillarbox = true;
    let xCount = 0, yCount = 0;

    // scan top row
    for (let i = 0; i < ROW_SIZE; i += PIXEL_SIZE) {
      if (
           imageData[i  ] > this.testResults.blackThreshold
        || imageData[i+1] > this.testResults.blackThreshold
        || imageData[i+2] > this.testResults.blackThreshold
      ) {
        imageData[i + 3] = GlDebugType.LetterboxOrientationScanImageDetection
        if (++xCount > xLimit) {
          letterbox = false;
          break;
        }
      } else {
        imageData[i+3] = GlDebugType.LetterboxOrientationScanTrace
      }
    }

    // scan sides
    for (let i = 0; i < imageSize; i += ROW_SIZE) {
      const lastPx = i + lastPixelOffset;

      // left side
      if (
           imageData[i  ] > this.testResults.blackThreshold
        || imageData[i+1] > this.testResults.blackThreshold
        || imageData[i+2] > this.testResults.blackThreshold
      ) {
        imageData[i+3] = GlDebugType.LetterboxOrientationScanImageDetection;
        if (++yCount > yLimit) {
          pillarbox = false;
          break;
        }
      } else {
        imageData[i+3] = GlDebugType.LetterboxOrientationScanTrace;
      }

      // right side
      if (
           imageData[lastPx  ] > this.testResults.blackThreshold
        || imageData[lastPx+1] > this.testResults.blackThreshold
        || imageData[lastPx+2] > this.testResults.blackThreshold
      ) {
        imageData[lastPx+3] = GlDebugType.LetterboxOrientationScanImageDetection;
        if (++yCount > yLimit) {
          pillarbox = false;
          break;
        }
      } else {
        imageData[lastPx+3] = GlDebugType.LetterboxOrientationScanTrace;
      }
    }

    // scan bottom row
    if (letterbox) {
      for (let i = ROW_SIZE * (height - 1); i < imageSize; i += PIXEL_SIZE) {
        if ( imageData[i  ] > this.testResults.blackThreshold
          || imageData[i+1] > this.testResults.blackThreshold
          || imageData[i+2] > this.testResults.blackThreshold
        ) {
          imageData[i + 3] = GlDebugType.LetterboxOrientationScanImageDetection
          if (++xCount > xLimit) {
            letterbox = false;
            break;
          }
        } else {
          imageData[i+3] = GlDebugType.LetterboxOrientationScanTrace
        }
      }
    }

    // determine result
    if (letterbox && pillarbox) {
      this.testResults.letterboxOrientation = LetterboxOrientation.Both;
    } else if (letterbox) {
      this.testResults.letterboxOrientation = LetterboxOrientation.Letterbox;
      this.testResults.lastValidLetterboxOrientation = LetterboxOrientation.Letterbox;
    } else if (pillarbox) {
      this.testResults.letterboxOrientation = LetterboxOrientation.Pillarbox;
      this.testResults.lastValidLetterboxOrientation = LetterboxOrientation.Pillarbox;
    } else {
      this.testResults.letterboxOrientation = LetterboxOrientation.NotLetterbox;
    }
  }

  /**
   * Updates letterbox edge (updates imageLine and guardLine)
   * @param crossDimension height of the sample frame (or width, if pillarbox)
   * @param topCandidate First line of image data on the top of  the frame
   * @param bottomCandidate First line with image data on the bottom of the frame
   * @returns
   */
  private updateLetterboxEdgeCandidates(crossDimension: number, topCandidate: number, bottomCandidate: number) {
    const bottomDistance = (crossDimension - bottomCandidate);
    const maxOffset = ~~(crossDimension * this.settings.active.arDetect.edgeDetection.maxLetterboxOffset);
    const diff = Math.abs(topCandidate - bottomDistance);
    const candidateAvg = ~~((topCandidate + bottomDistance) * 0.5);

    this.testResults.aspectRatioCheck.frontCandidate = topCandidate;
    this.testResults.aspectRatioCheck.backCandidate = bottomCandidate;

    if (diff > maxOffset) {
      this.testResults.aspectRatioUncertain = true;
      this.testResults.aspectRatioUncertainReason = AardUncertainReason.LetterboxNotCenteredEnough;
    }

    this.testResults.letterboxSize = candidateAvg;
    this.testResults.letterboxOffset = diff;
  }


  /**
   * Scans for subtitles
   * @param imageData
   * @param width
   * @param height
   * @returns
   */
   private subtitleScan(imageData: Uint8Array, width: number, height: number, skipAdvancedScan: boolean) {
    const scanConf = this.settings.active.arDetect.subtitles;

    this.testResults.subtitleDetected = false;

    // SubtitleScanResult Regions
    const ssrRegions = this.testResults.subtitleScan.regions;
    const halfHeight = Math.floor(height / 2);

    resetSubtitleScanResults(this.testResults);

    this.subtitleScanRegionIterative(
      imageData, height,
      2, halfHeight,
      scanConf.refiningScanSpacing, scanConf.minDetections,
      ssrRegions.top,
    )

    this.subtitleScanRegionIterative(
      imageData, height,
      height - 3, halfHeight,
      -scanConf.refiningScanSpacing, scanConf.minDetections,
      ssrRegions.bottom,
    )


    if (ssrRegions.top.uncertain || ssrRegions.bottom.uncertain) {
      this.testResults.aspectRatioUncertain = true;
    } else {
      this.testResults.aspectRatioUncertain = false;
    }
    if (ssrRegions.top.firstSubtitle !== -1 || ssrRegions.bottom.firstSubtitle !== -1) {
      this.testResults.subtitleDetected = true;
    }

    this.updateLetterboxEdgeCandidates(
      height,
      ssrRegions.top.firstImage,
      ssrRegions.bottom.firstImage
    );

    this.timer.current.subtitleScan = performance.now() - this.timer.current.start;
  }

  /**
   * Scans region of video frame for presence of subtitles
   * @param imageData
   * @param startRow
   * @param endRow
   * @param minDetections
   * @param results
   */
  private subtitleScanRegionLinear(
    imageData: Uint8Array,
    height: number,
    startRow: number,
    endRow: number,
    scanSpacing: number,
    minDetections: number,
    results: AardTestResult_SubtitleRegion,
  ) {
    results.uncertain = false;

    const scanConf = this.settings.active.arDetect.subtitles;
    const arConf = this.settings.active.arDetect;

    let letterCount, imageSegmentCount, potentialFadedLetterCount, potentialFadedLetterCountInvalidated, nonGradientPixelCount, letterSize, imageSize, imageSegmentSize, imageWeightedSize, segmentWeights, imageSegmentAlignment, imageSegmentAlignmentSamples,
      isOnLetter, isOnImage, isBlank,
      gradientRowOffset_before, gradientRowOffset_after;
    let rowStart, rowEnd, rowMid, rowGTA, rowGTB; // GT = gradient test
    let imageConfirmPass = false, subtitleConfirmPass = false;

    let innerIteration, outerIteration = 0;

    const rowMargin = Math.floor(scanConf.scanMargin * ROW_SIZE);
    const imageThreshold = Math.floor((ROW_SIZE - (rowMargin * 2)) * arConf.edgeDetection.minValidImage * PIXEL_SIZE_FRACTION);


    // search in top letterbox
    outerLoop:
    for (
      let searchRow = startRow;
      (scanSpacing > 0 && searchRow < endRow) || (scanSpacing < 0 && searchRow > endRow);
      searchRow += scanSpacing
    ) {
      innerIteration = 0;
      outerIteration++;

      letterCount = 0;
      potentialFadedLetterCount = 0;
      imageSegmentCount = 0;
      imageSegmentSize = 0;
      nonGradientPixelCount = 0;

      imageSize = 0;
      imageWeightedSize = 0;
      segmentWeights = 0;

      letterSize = 0;
      isOnLetter = false;
      isOnImage = false;
      isBlank = true;
      potentialFadedLetterCountInvalidated = false;

      imageSegmentAlignment = 0;
      imageSegmentAlignmentSamples = 0;

      // Scan region is centered,
      rowStart = (searchRow * ROW_SIZE) + rowMargin;

      if (scanSpacing > 0) {
        gradientRowOffset_before = (Math.max(searchRow - 2, 0)      * ROW_SIZE);
        gradientRowOffset_after  = (Math.min(searchRow + 2, height) * ROW_SIZE);
      } else {
        gradientRowOffset_before = (Math.max(searchRow - 2, 0)      * ROW_SIZE);
        gradientRowOffset_after  = (Math.min(searchRow + 2, height) * ROW_SIZE);
      }

      // exact row doesn't matter ... unless scanMargin is 0
      rowEnd = ((searchRow + 1) * ROW_SIZE) - rowMargin;
      rowMid = (rowStart + rowEnd) * 0.5;

      const resetSubtitlePass =() => {
        if (subtitleConfirmPass) {
          subtitleConfirmPass = false;
          searchRow -= (scanSpacing > 0 ? + 1 : -1);
        }
      }

      const updateImage = (candidate: number) => {
        if (scanSpacing > 0) {
          if (results.firstImage === -1 || candidate < results.firstImage) {
            results.firstImage = candidate;
          }
        } else {
          if (results.firstImage === -1 || candidate > results.firstImage) {
            results.firstImage = candidate;
          }
        }
      }

      /**
       * This function can break or return early only in the following situations:
       *
       *     * we have enough letters to detect subtitles — continue outerLoop
       *     * Single "letter" is too wide — immediate return, as we found where the image starts
       *
       * Other instances require a bit more complex analysis.
       */

      while (rowStart < rowEnd) {
        innerIteration++;

        const r = imageData[rowStart], g = imageData[rowStart + 1], b = imageData[rowStart + 2];

        const on = r > scanConf.subtitleSubpixelThresholdOn
                    || g > scanConf.subtitleSubpixelThresholdOn
                    || b > scanConf.subtitleSubpixelThresholdOn;
        const off = r < scanConf.subtitleSubpixelThresholdOff
                    && g < scanConf.subtitleSubpixelThresholdOff
                    && b < scanConf.subtitleSubpixelThresholdOff;

        if (off) {
          imageData[rowStart + 3] = GlDebugType.SubtitleThresholdOff;


          // if isOnLetter was set, that means we've just concluded a letter segment
          if (isOnLetter) {
            letterCount++;

            if (letterCount > minDetections) {
              if (results.firstSubtitle === -1) {
                results.firstSubtitle = searchRow;

                // if detecting subtitles only resets AR, we can return immediately
                if (scanConf.subtitleCropMode !== AardSubtitleCropMode.CropSubtitles) {
                  break outerLoop;
                }
              }
              results.lastSubtitle = searchRow;
              isBlank = false;

              imageConfirmPass = false;
              continue outerLoop;
            }
          }
          if (isOnImage) {
            if (imageSegmentSize < scanConf.maxValidLetter) {
              potentialFadedLetterCount++;

              // track potential letter alignment
              imageSegmentAlignment += (rowStart - rowMid) * imageSegmentSize * PIXEL_SIZE_FRACTION;
              imageSegmentAlignmentSamples += imageSegmentSize;
            } else {
              potentialFadedLetterCountInvalidated = true;
            }

            if (imageSegmentSize > arConf.edgeDetection.minEdgeSegmentSize) {
              imageSegmentCount++;
              imageWeightedSize += imageSegmentSize * imageSegmentSize; // longer segments should have bigger weight
              segmentWeights += imageSegmentSize;
              imageSegmentSize = 0;
            }
          }

          isOnLetter = false;
          isOnImage = false;
          letterSize = 0;
          imageSegmentSize = 0;
        } else {
          imageData[rowStart + 3] = GlDebugType.SubtitleThresholdNone;
          isOnImage = true;
          isBlank = false;
          imageSegmentSize++;
          imageSize++;

          // let's see if we're accidentally detecting gradient.
          // We only need to run these tests when pixels are dark.
          // We keep track of number of pixels that pass the test, or don't
          // need the test to begin with.
          if (!on && (
               imageData[rowStart    ] < arConf.edgeDetection.gradientThreshold
            && imageData[rowStart + 1] < arConf.edgeDetection.gradientThreshold
            && imageData[rowStart + 2] < arConf.edgeDetection.gradientThreshold
          )) {
            rowGTB = rowStart - gradientRowOffset_before;
            rowGTA = rowStart + gradientRowOffset_after;

            // if true, then gradient.
            // The first row gives technically incorrect answers for pixels directly under subtitles, but since
            // "nonGradientPixelCount" is a shorthand for "we're relatively confident in this detection", potentially
            // mistaking non-black pixels under subtitles for gradient isn't too problematic
            if ( (imageData[rowStart    ] - imageData[rowGTB    ]) < arConf.edgeDetection.gradientTestMinDelta
              && (imageData[rowStart + 1] - imageData[rowGTB + 1]) < arConf.edgeDetection.gradientTestMinDelta
              && (imageData[rowStart + 2] - imageData[rowGTB + 2]) < arConf.edgeDetection.gradientTestMinDelta
              && imageData[rowGTA    ] - imageData[rowStart    ] > arConf.edgeDetection.gradientTestMinDeltaAfter
              && imageData[rowGTA + 1] - imageData[rowStart + 1] > arConf.edgeDetection.gradientTestMinDeltaAfter
              && imageData[rowGTA + 2] - imageData[rowStart + 2] > arConf.edgeDetection.gradientTestMinDeltaAfter
              && imageData[rowGTA    ] - imageData[rowStart    ] < arConf.edgeDetection.gradientTestMaxDeltaAfter
              && imageData[rowGTA + 1] - imageData[rowStart + 1] < arConf.edgeDetection.gradientTestMaxDeltaAfter
              && imageData[rowGTA + 2] - imageData[rowStart + 2] < arConf.edgeDetection.gradientTestMaxDeltaAfter
            ) {
              // we do nothing (at this moment)
            } else {
              nonGradientPixelCount++;
            }
          } else {
            nonGradientPixelCount++;
          }
        }
        if (on) {   // used to detect subtitles specifically
          imageData[rowStart + 3] = GlDebugType.SubtitleThresholdOn;
          isOnLetter = true;
          letterSize++;

          // bail on invalid letter sizes — this means we're seeing image.
          // in this case, we do not need image confirmation step
          if (letterSize > scanConf.maxValidLetter) {
            updateImage(searchRow);
            isBlank = false;

            return;
          }
        }

        rowStart += PIXEL_SIZE;
      }


      // we need to do this once more, otherwise imageSegmentCount can be 0 & bugs happen
      if (isOnImage) {
        if (imageSegmentSize < scanConf.maxValidLetter) {
          potentialFadedLetterCount++;

          // track potential letter alignment
          imageSegmentAlignment += (rowStart - rowMid) * imageSegmentSize * PIXEL_SIZE_FRACTION;
          imageSegmentAlignmentSamples += imageSegmentSize;
        } else {
          potentialFadedLetterCountInvalidated = true;
        }

        if (imageSegmentSize > arConf.edgeDetection.minEdgeSegmentSize) {
          imageSegmentCount++;
          imageWeightedSize += imageSegmentSize * imageSegmentSize; // longer segments should have bigger weight
          segmentWeights += imageSegmentSize;
          imageSegmentSize = 0;
        }
      }


      if (!imageSegmentCount || isBlank) {
        isBlank = true;
        imageConfirmPass = false;

        if (results.firstBlank === -1) {
          results.firstBlank = searchRow;
        }
        results.lastBlank = searchRow;

        resetSubtitlePass();
        continue outerLoop;
      }

      const averageImageSegmentSize = segmentWeights > 0 ? imageWeightedSize / segmentWeights : 0;
      const gradientDetectionFrequency = 1 - (nonGradientPixelCount / imageSize);

      // That's probably a subtitle as well. If subs are fading in and out,
      // then there's a good chance that they won't meet the "yes, this is a letter" threshold.
      if (
        !potentialFadedLetterCountInvalidated
        && potentialFadedLetterCount > scanConf.minDetections
        // FOR SOME REASON THIS CONDITION CAUSES EVERYTHING TO HANG:
        // && (imageSegmentAlignmentSamples && imageSegmentAlignment && Math.abs(imageSegmentAlignment / imageSegmentAlignmentSamples) > scanConf.maxPotentialSubtitleMisalignment)
        && averageImageSegmentSize < scanConf.maxValidLetter
      ) {
        if (subtitleConfirmPass) {
          if (results.firstSubtitle === -1) {
            results.firstSubtitle = searchRow;

            // if detecting subtitles only resets AR, we can return immediately
            if (scanConf.subtitleCropMode !== AardSubtitleCropMode.CropSubtitles) {
              break outerLoop;
            }
          }
          results.lastSubtitle = searchRow;
          isBlank = false;

          resetSubtitlePass();
        }
        subtitleConfirmPass = true;
        searchRow -= scanSpacing + (scanSpacing > 0 ? - 1 : 1);
        continue outerLoop;
      }

      // If we are here, subtitles weren't confirmed
      resetSubtitlePass();


      // If we detect gradient, that's instant fail.
      // We still save uncertain detection to firstImage, because iterative scan uses that
      // in order to determine which region to scan further
      if (gradientDetectionFrequency > arConf.edgeDetection.gradientThreshold) {
        results.uncertain = true;
        updateImage(searchRow);
        break outerLoop;
      }

      // Cases which require confirmation
      if (
        imageSegmentCount > arConf.edgeDetection.maxEdgeSegments // we need to confirm if there's too many segments
        || averageImageSegmentSize < arConf.edgeDetection.averageEdgeThreshold // we also need to confirm if segments are too small
      ) {
        if (imageConfirmPass) {
          results.uncertain = true;
          updateImage(searchRow - 1);
          break outerLoop;
        }

        imageConfirmPass = true;
        // imageConfirmPass must happen on the next row, but it's possible that we aren't
        // checking row-by-row. Hence, we need to modify our scan a bit
        searchRow -= scanSpacing + (scanSpacing > 0 ? - 1 : 1);

        continue outerLoop;
      }

      if (averageImageSegmentSize > arConf.edgeDetection.averageEdgeThreshold || imageSize > imageThreshold) {
        updateImage(searchRow);
        break outerLoop;
      }

      // we can only reach this far if we detected image
      if (imageConfirmPass) {
        updateImage(searchRow - 1);
      } else {
        updateImage(searchRow);
      }
      break outerLoop;
    } // end of outer loop
  }

  /**
   * Tries to determine letterbox through subtitles
   * @param imageData
   * @param startRow
   * @param endRow
   * @param ROW_SIZE
   * @param scanSpacing
   * @param minDetections
   * @param results
   * @param ssrRegionName
   * @returns
   */
  private subtitleScanRegionIterative(
    imageData: Uint8Array,
    height: number,
    startRow: number,
    endRow: number,
    scanSpacing: number,
    minDetections: number,
    results: AardTestResult_SubtitleRegion,
  ): boolean {
    while (true) {
      if (scanSpacing > -1 && scanSpacing < 1) {
        break;
      }
      this.subtitleScanRegionLinear(
        imageData, height,
        startRow, endRow,
        scanSpacing, minDetections, results
      );

      if (results.firstImage === -1) {
        return false;
      }

      // We need to ensure small amount of overlap, in case we landed on a sus row
      if (scanSpacing > 0) {
        startRow = Math.max(results.firstImage - Math.floor(scanSpacing * 2), 0);
        endRow = Math.min(results.firstImage + Math.floor(scanSpacing * 2), height - 1);
      } else {
        startRow = Math.min(results.firstImage + Math.floor(-scanSpacing * 2), height - 1);
        endRow = Math.max(results.firstImage - Math.floor(-scanSpacing * 2), 0);
      }

      scanSpacing = scanSpacing / 2;
    }

    return true;
  }



  /**
   * Updates aspect ratio if new aspect ratio is different enough from the old one
   */
  private updateAspectRatio(ar: number, options?: {uncertainDetection?: boolean, forceReset?: boolean}) {
    // Calculate difference between two ratios

    // We need to detect updates even if subtitles are detected — we just don't trigger
    // the actual aspect ratio change if everything is paused.
    if (this.timers.pauseUntil > Date.now() && !options?.forceReset) {
      return false;
    }

    const maxRatio = Math.max(ar, this.testResults.activeAspectRatio);
    const diff = Math.abs(ar - this.testResults.activeAspectRatio);

    if ((diff / maxRatio) > this.settings.active.arDetect.allowedArVariance || options?.forceReset) {
      this.videoData.resizer.updateAr({
        type: AspectRatioType.AutomaticUpdate,
        ratio: ar,
        offset: this.testResults.letterboxOffset,
        variant: this.arVariant
      });
      this.testResults.activeAspectRatio = ar;

      if (!options?.uncertainDetection) {
        if (this.settings.active.arDetect.autoDisable.onFirstChange) {
          this.status.autoDisabled = true;
        }
        if (this.settings.active.arDetect.autoDisable.ifNotChanged) {
          this.timers.autoDisableAt = Date.now() + this.settings.active.arDetect.autoDisable.ifNotChangedTimeout;
        }
      }

      this.testResults.aspectRatioUpdated = true;
    }
  }

  /**
   * Calculates video's current aspect ratio based on data in testResults.
   * @returns
   */
  private getAr(): number {
    const fileAr = this.video.videoWidth / this.video.videoHeight;
    const canvasAr = this.canvasStore.main.width / this.canvasStore.main.height;

    const compensatedWidth = fileAr === canvasAr ? this.canvasStore.main.width : this.video.videoWidth * this.canvasStore.main.height / (this.video.videoHeight);

    // console.log(`
    //   ———— ASPECT RATIO CALCULATION: —————

    //   canvas size: ${this.canvasStore.main.width} x ${this.canvasStore.main.height} (1:${this.canvasStore.main.width / this.canvasStore.main.height})
    //   file size: ${this.video.videoWidth} x ${this.video.videoHeight} (1:${this.video.videoWidth / this.video.videoHeight})

    //   compensated size: ${compensatedWidth} x ${this.canvasStore.main.height} (1:${compensatedWidth / this.canvasStore.main.height})

    //   letterbox height: ${this.testResults.letterboxWidth}
    //   net video height: ${this.canvasStore.main.height - (this.testResults.letterboxWidth * 2)}

    //   calculated aspect ratio -----

    //          ${compensatedWidth}               ${compensatedWidth}         ${compensatedWidth}
    //     ——————————————— = —————————————— = —————— =  ${compensatedWidth / (this.canvasStore.main.height - (this.testResults.letterboxWidth * 2))}
    //      ${this.canvasStore.main.height} - 2 x ${this.testResults.letterboxWidth}       ${this.canvasStore.main.height} - ${2 * this.testResults.letterboxWidth}       ${this.canvasStore.main.height - (this.testResults.letterboxWidth * 2)}
    // `);

    if (this.testResults.letterboxOrientation === LetterboxOrientation.Pillarbox) {
      const compensationFactor = compensatedWidth / this.canvasStore.main.width;
      const pillarboxCompensated = (this.testResults.letterboxSize * 2 * compensationFactor);
      return (compensatedWidth - pillarboxCompensated) / this.canvasStore.main.height;
    } else {
      const heightWithoutLetterbox = this.canvasStore.main.height - (this.testResults.letterboxSize * 2);
      return compensatedWidth / heightWithoutLetterbox;
    }
  }

  //#endregion

}