Finish aspect ratio check

src/common/interfaces/SettingsInterface.ts

@@ -51,139 +51,165 @@ export interface CommandInterface {
 export type SettingsReloadComponent = 'PlayerData' | 'VideoData';
 export type SettingsReloadFlags = true | SettingsReloadComponent;
 
+export interface AardSettings {
+  disabledReason: string,     // if automatic aspect ratio has been disabled, show reason
+  allowedMisaligned: number,  // top and bottom letterbox thickness can differ by this much.
+                              // Any more and we don't adjust ar.
+  allowedArVariance: number,  // amount by which old ar can differ from the new (1 = 100%)
+  timers: {                   // autodetection frequency
+    playing: number,            // while playing
+    paused: number,             // while paused
+    error: number,              // after error
+    minimumTimeout: number,
+    tickrate: number,          // 1 tick every this many milliseconds
+  },
+  autoDisable: {            // settings for automatically disabling the extension
+    maxExecutionTime: number, // if execution time of main autodetect loop exceeds this many milliseconds,
+                            // we disable it.
+    consecutiveTimeoutCount: number,  // we only do it if it happens this many consecutive times
+
+    // FOR FUTURE USE
+    consecutiveArResets: number       // if aspect ratio reverts immediately after AR change is applied, we disable everything
+  },
+  canvasDimensions: {
+    blackframeCanvas: {   // smaller than sample canvas, blackframe canvas is used to recon for black frames
+                          // it's not used to detect aspect ratio by itself, so it can be tiny af
+      width: number,
+      height: number,
+    },
+    sampleCanvas: {   // size of image sample for detecting aspect ratio. Bigger size means more accurate results,
+                          // at the expense of performance
+      width: number,
+      height: number,
+    },
+  },
+
+  // NOTE: Black Frame is currently not in use.
+  blackframe: {
+    sufficientColorVariance: number,  // calculate difference between average intensity and pixel, for every pixel for every color
+                                    // component. Average intensity is normalized to where 0 is black and 1 is biggest value for
+                                    // that component. If sum of differences between normalized average intensity and normalized
+                                    // component varies more than this % between color components, we can afford to use less strict
+                                    // cumulative threshold.
+    cumulativeThresholdLax: number,
+    cumulativeThresholdStrict: number,// if we add values of all pixels together and get more than this, the frame is bright enough.
+                               // (note: blackframe is 16x9 px -> 144px total. cumulative threshold can be reached fast)
+    blackPixelsCondition: number, // How much pixels must be black (1 all, 0 none) before we consider frame as black. Takes
+                               // precedence over cumulative threshold: if blackPixelsCondition is met, the frame is dark
+                               // regardless of whether cumulative threshold has been reached.
+  },
+
+  // Used by old aspect ratio detection algorithm. Pls remove.
+  blackbar: {
+    blackLevel: number,         // everything darker than 10/255 across all RGB components is considered black by
+                            // default. blackLevel can decrease if we detect darker black.
+    threshold: number,          // if pixel is darker than the sum of black level and this value, we count it as black
+                            // on 0-255. Needs to be fairly high (8 might not cut it) due to compression
+                            // artifacts in the video itself
+    frameThreshold: number,      // threshold, but when doing blackframe test
+    imageThreshold: number,     // in order to detect pixel as "not black", the pixel must be brighter than
+                            // the sum of black level, threshold and this value.
+    gradientThreshold: number,   // When trying to determine thickness of the black bars, we take 2 values: position of
+                            // the last pixel that's darker than our threshold, and position of the first pixel that's
+                            // brighter than our image threshold. If positions are more than this many pixels apart,
+                            // we assume we aren't looking at letterbox and thus don't correct the aspect ratio.
+    gradientSampleSize: number, // How far do we look to find the gradient
+    maxGradient: number,         // if two neighboring pixels in gradientSampleSize differ by more than this, then we aren't
+                            // looking at a gradient
+    gradientNegativeTreshold: number,
+    gradientMaxSD: number,    // reserved for future use
+    antiGradientMode: AntiGradientMode
+  },
+  // Also not in use, probs.
+  variableBlackbarThresholdOptions: {    // In case of poor bitrate videos, jpeg artifacts may cause us issues
+    // FOR FUTURE USE
+    enabled: boolean,                      // allow increasing blackbar threshold
+    disableArDetectOnMax: boolean,         // disable autodetection when threshold goes over max blackbar threshold
+    maxBlackbarThreshold: number,            // max threshold (don't increase past this)
+    thresholdStep: number,                   // when failing to set aspect ratio, increase threshold by this much
+    increaseAfterConsecutiveResets: number   // increase if AR resets this many times in a row
+  },
+  blackLevels: {
+    defaultBlack: number,    // By default, pixels darker than this are considered black.
+                             // (If detection algorithm detects darker blacks, black is considered darkest detected pixel)
+    blackTolerance: number,  // If pixel is more than this much brighter than blackLevel, it's considered not black
+                             // It is not considered a valid image detection if gradient detection is enabled
+    imageDelta: number,      // When gradient detection is enabled, pixels this much brighter than black skip gradient detection
+  }
+  sampling: {
+    edgePosition: number;    // % of width (max 0.33). Pixels up to this far away from either edge may contain logo.
+    staticCols: number,      // we take a column at [0-n]/n-th parts along the width and sample it
+    randomCols: number,      // we add this many randomly selected columns to the static columns
+    staticRows: number,      // forms grid with staticSampleCols. Determined in the same way. For black frame checks,
+  },
+  guardLine: {              // all pixels on the guardline need to be black, or else we trigger AR recalculation
+                            // (if AR fails to be recalculated, we reset AR)
+    enabled: boolean,
+    ignoreEdgeMargin: number, // we ignore anything that pokes over the black line this close to the edge
+                            // (relative to width of the sample)
+    imageTestThreshold: number, // when testing for image, this much pixels must be over blackbarThreshold
+    edgeTolerancePx: number,         // black edge violation is performed this far from reported 'last black pixel'
+    edgeTolerancePercent: null  // unused. same as above, except use % of canvas height instead of pixels
+  },
+  arSwitchLimiter: {          // to be implemented
+    switches: number,              // we can switch this many times
+      period: number             // per this period
+  },
+
+
+  // pls deprecate and move things used
+  edgeDetection: {
+    slopeTestWidth: number,
+    gradientTestSamples: number,         // we check this many pixels below (or above) the suspected edge to check for gradient
+    gradientTestBlackThreshold: number,  // if pixel in test sample is brighter than that, we aren't looking at gradient
+    gradientTestDeltaThreshold: number,  // if delta between two adjacent pixels in gradient test exceeds this, it's not gradient
+    gradientTestMinDelta: number,        // if last pixels of the test sample is less than this brighter than the first -> not gradient
+
+    thresholds: {
+      edgeDetectionLimit: 8,             // during scanning of the edge, quit after edge gets detected at this many points
+      minQualitySingleEdge: 6,           // At least one of the detected must reach this quality
+      minQualitySecondEdge: 3,           // The other edge must reach this quality (must be smaller or equal to single edge quality)
+    }
+
+    maxLetterboxOffset: 0.1,             // Upper and lower letterbox can be different by this many (% of height)
+
+    // Previous iteration variables VVVV
+    sampleWidth: number,        // we take a sample this wide for edge detection
+    detectionThreshold: number,  // sample needs to have this many non-black pixels to be a valid edge
+    confirmationThreshold: number,  //
+    singleSideConfirmationThreshold: number,    // we need this much edges (out of all samples, not just edges) in order
+                                           // to confirm an edge in case there's no edges on top or bottom (other
+                                          // than logo, of course)
+    logoThreshold: number,     // if edge candidate sits with count greater than this*all_samples, it can't be logo
+                            // or watermark.
+    edgeTolerancePx?: number,          // we check for black edge violation this far from detection point
+    edgeTolerancePercent?: number,  // we check for black edge detection this % of height from detection point. unused
+    middleIgnoredArea: number,      // we ignore this % of canvas height towards edges while detecting aspect ratios
+    minColsForSearch: number,       // if we hit the edge of blackbars for all but this many columns (%-wise), we don't
+                                 // continue with search. It's pointless, because black edge is higher/lower than we
+                                 // are now. (NOTE: keep this less than 1 in case we implement logo detection)
+  },
+  pillarTest: {
+    ignoreThinPillarsPx: number, // ignore pillars that are less than this many pixels thick.
+    allowMisaligned: number   // left and right edge can vary this much (%)
+  },
+  textLineTest: {
+    nonTextPulse: number,     // if a single continuous pulse has this many non-black pixels, we aren't dealing
+                            // with text. This value is relative to canvas width (%)
+    pulsesToConfirm: number,    // this is a threshold to confirm we're seeing text.
+    pulsesToConfirmIfHalfBlack: number, // this is the threshold to confirm we're seeing text if longest black pulse
+                                   // is over 50% of the canvas width
+    testRowOffset: number     // we test this % of height from detected edge
+  }
+}
+
 interface SettingsInterface {
   _updateFlags?: {
     requireReload?: SettingsReloadFlags,
     forSite?: string
   }
 
-  arDetect: {
+  arDetect: AardSettings,
-    disabledReason: string,     // if automatic aspect ratio has been disabled, show reason
-    allowedMisaligned: number,  // top and bottom letterbox thickness can differ by this much.
-                                // Any more and we don't adjust ar.
-    allowedArVariance: number,  // amount by which old ar can differ from the new (1 = 100%)
-    timers: {                   // autodetection frequency
-      playing: number,            // while playing
-      paused: number,             // while paused
-      error: number,              // after error
-      minimumTimeout: number,
-      tickrate: number,          // 1 tick every this many milliseconds
-    },
-    autoDisable: {            // settings for automatically disabling the extension
-      maxExecutionTime: number, // if execution time of main autodetect loop exceeds this many milliseconds,
-                              // we disable it.
-      consecutiveTimeoutCount: number,  // we only do it if it happens this many consecutive times
-
-      // FOR FUTURE USE
-      consecutiveArResets: number       // if aspect ratio reverts immediately after AR change is applied, we disable everything
-    },
-    canvasDimensions: {
-      blackframeCanvas: {   // smaller than sample canvas, blackframe canvas is used to recon for black frames
-                            // it's not used to detect aspect ratio by itself, so it can be tiny af
-        width: number,
-        height: number,
-      },
-      sampleCanvas: {   // size of image sample for detecting aspect ratio. Bigger size means more accurate results,
-                            // at the expense of performance
-        width: number,
-        height: number,
-      },
-    },
-
-    // samplingInterval: 10,     // we sample at columns at (width/this) * [ 1 .. this - 1]
-    blackframe: {
-      sufficientColorVariance: number,  // calculate difference between average intensity and pixel, for every pixel for every color
-                                      // component. Average intensity is normalized to where 0 is black and 1 is biggest value for
-                                      // that component. If sum of differences between normalized average intensity and normalized
-                                      // component varies more than this % between color components, we can afford to use less strict
-                                      // cumulative threshold.
-      cumulativeThresholdLax: number,
-      cumulativeThresholdStrict: number,// if we add values of all pixels together and get more than this, the frame is bright enough.
-                                 // (note: blackframe is 16x9 px -> 144px total. cumulative threshold can be reached fast)
-      blackPixelsCondition: number, // How much pixels must be black (1 all, 0 none) before we consider frame as black. Takes
-                                 // precedence over cumulative threshold: if blackPixelsCondition is met, the frame is dark
-                                 // regardless of whether cumulative threshold has been reached.
-    },
-    blackbar: {
-      blackLevel: number,         // everything darker than 10/255 across all RGB components is considered black by
-                              // default. blackLevel can decrease if we detect darker black.
-      threshold: number,          // if pixel is darker than the sum of black level and this value, we count it as black
-                              // on 0-255. Needs to be fairly high (8 might not cut it) due to compression
-                              // artifacts in the video itself
-      frameThreshold: number,      // threshold, but when doing blackframe test
-      imageThreshold: number,     // in order to detect pixel as "not black", the pixel must be brighter than
-                              // the sum of black level, threshold and this value.
-      gradientThreshold: number,   // When trying to determine thickness of the black bars, we take 2 values: position of
-                              // the last pixel that's darker than our threshold, and position of the first pixel that's
-                              // brighter than our image threshold. If positions are more than this many pixels apart,
-                              // we assume we aren't looking at letterbox and thus don't correct the aspect ratio.
-      gradientSampleSize: number, // How far do we look to find the gradient
-      maxGradient: number,         // if two neighboring pixels in gradientSampleSize differ by more than this, then we aren't
-                              // looking at a gradient
-      gradientNegativeTreshold: number,
-      gradientMaxSD: number,    // reserved for future use
-      antiGradientMode: AntiGradientMode
-    },
-    variableBlackbarThresholdOptions: {    // In case of poor bitrate videos, jpeg artifacts may cause us issues
-      // FOR FUTURE USE
-      enabled: boolean,                      // allow increasing blackbar threshold
-      disableArDetectOnMax: boolean,         // disable autodetection when threshold goes over max blackbar threshold
-      maxBlackbarThreshold: number,            // max threshold (don't increase past this)
-      thresholdStep: number,                   // when failing to set aspect ratio, increase threshold by this much
-      increaseAfterConsecutiveResets: number   // increase if AR resets this many times in a row
-    },
-    sampling: {
-      staticCols: number,      // we take a column at [0-n]/n-th parts along the width and sample it
-      randomCols: number,      // we add this many randomly selected columns to the static columns
-      staticRows: number,      // forms grid with staticSampleCols. Determined in the same way. For black frame checks
-    },
-    guardLine: {              // all pixels on the guardline need to be black, or else we trigger AR recalculation
-                              // (if AR fails to be recalculated, we reset AR)
-      enabled: boolean,
-      ignoreEdgeMargin: number, // we ignore anything that pokes over the black line this close to the edge
-                              // (relative to width of the sample)
-      imageTestThreshold: number, // when testing for image, this much pixels must be over blackbarThreshold
-      edgeTolerancePx: number,         // black edge violation is performed this far from reported 'last black pixel'
-      edgeTolerancePercent: null  // unused. same as above, except use % of canvas height instead of pixels
-    },
-    fallbackMode: {
-      enabled: boolean,
-      safetyBorderPx: number,        // determines the thickness of safety border in fallback mode
-      noTriggerZonePx: number        // if we detect edge less than this many pixels thick, we don't correct.
-    },
-    arSwitchLimiter: {          // to be implemented
-      switches: number,              // we can switch this many times
-        period: number             // per this period
-    },
-    edgeDetection: {
-      sampleWidth: number,        // we take a sample this wide for edge detection
-      detectionThreshold: number,  // sample needs to have this many non-black pixels to be a valid edge
-      confirmationThreshold: number,  //
-      singleSideConfirmationThreshold: number,    // we need this much edges (out of all samples, not just edges) in order
-                                             // to confirm an edge in case there's no edges on top or bottom (other
-                                            // than logo, of course)
-      logoThreshold: number,     // if edge candidate sits with count greater than this*all_samples, it can't be logo
-                              // or watermark.
-      edgeTolerancePx?: number,          // we check for black edge violation this far from detection point
-      edgeTolerancePercent?: number,  // we check for black edge detection this % of height from detection point. unused
-      middleIgnoredArea: number,      // we ignore this % of canvas height towards edges while detecting aspect ratios
-      minColsForSearch: number,       // if we hit the edge of blackbars for all but this many columns (%-wise), we don't
-                                   // continue with search. It's pointless, because black edge is higher/lower than we
-                                   // are now. (NOTE: keep this less than 1 in case we implement logo detection)
-    },
-    pillarTest: {
-      ignoreThinPillarsPx: number, // ignore pillars that are less than this many pixels thick.
-      allowMisaligned: number   // left and right edge can vary this much (%)
-    },
-    textLineTest: {
-      nonTextPulse: number,     // if a single continuous pulse has this many non-black pixels, we aren't dealing
-                              // with text. This value is relative to canvas width (%)
-      pulsesToConfirm: number,    // this is a threshold to confirm we're seeing text.
-      pulsesToConfirmIfHalfBlack: number, // this is the threshold to confirm we're seeing text if longest black pulse
-                                     // is over 50% of the canvas width
-      testRowOffset: number     // we test this % of height from detected edge
-    }
-  },
 
   ui: {
     inPlayer: {

src/ext/lib/aard/interfaces/aard-detection-sample.interface.ts

@@ -0,0 +1,77 @@
+/**
+ * Used to store coordinates of sample columns/rows and the
+ * first x/y position where non-black pixels were detected.
+ *
+ * Arrays are laid out like so:
+ *
+ *    We check each row at these positions (columns)
+ *     V         V        V
+ *    _________ _________ _________ _____
+ *   | x₀ | y₀ | x₁ | y₁ | x₂ | y₂ | ..
+ *   '''''''''' ''''''''' ''''''''' '''''
+ *          A         A         A
+ *   If checked pixel is non-black, we put current row into
+ *   this element of the array.
+ *
+ */
+export interface AardDetectionSample {
+  top?: Int16Array;
+  bottom?: Int16Array;
+  left?: Int16Array;
+  right?: Int16Array;
+}
+
+export function generateSampleArray(samples: number, width: number, topBottom: boolean = true) {
+  const sampleStore = new Int16Array(samples * 2);
+
+  /**
+   * We want to reverse-fencepost here.
+   *
+   * Normally, our sample positions would look like this:
+   *
+   *
+   * 0    1    2    3
+   * |                   :
+   * |————|————|————|————:
+   * | <——  20 units ——> :
+   * 0                   19
+   *
+   * But we'd rather our samples are center-justified.
+   * We can solve this issue by dividing the width into
+   * (samples + 1) slices, and ignoring the first (0)
+   * position:
+   *
+   *     0   1   2   3
+   * :
+   * :———|———|———|———|———:
+   * :                   :
+   * 0                   19
+   *
+   */
+  const sampleInterval = ~~(width / ( samples + 1 ));
+
+  let i = 0, col = 1;
+  while (i < sampleStore.length) {
+    sampleStore[i] = sampleInterval * col * (+topBottom * 4);
+    i++;
+    // initialize to -1 (invalid result)
+    sampleStore[i] = -1;
+    i++;
+    col++;
+  }
+
+  return sampleStore;
+}
+
+export function resetSamples(samples: AardDetectionSample) {
+  samples.top && resetArray(samples.top);
+  samples.bottom && resetArray(samples.bottom);
+  samples.left && resetArray(samples.left);
+  samples.right && resetArray(samples.right);
+}
+
+function resetArray(x: Int16Array) {
+  for (let i = 1; i < x.length; i+= 2) {
+    x[i] = -1;
+  }
+}

src/ext/lib/aard/interfaces/aard-gradient-sample.interface.ts

@@ -0,0 +1,54 @@
+export interface AardGradientSamples {
+  top: Array<Uint8Array>,
+  bottom: Array<Uint8Array>,
+  left?: Array<Uint8Array>,
+  right?: Array<Uint8Array>,
+}
+
+export interface AardGradientSampleOptions {
+  aspectRatioSamples: number;
+  gradientSamples: number,
+}
+
+function generateArray(samplingOptions: AardGradientSampleOptions) {
+  const arr = new Array<Uint8Array>(samplingOptions.aspectRatioSamples)
+  for (let i = 0; i < samplingOptions.aspectRatioSamples; i++) {
+    arr[i] = new Uint8Array(samplingOptions.gradientSamples);
+  }
+  return arr;
+}
+
+export function initAardGradientSamples(letterboxSamplingOptions: AardGradientSampleOptions): AardGradientSamples {
+  return {
+    top: generateArray(letterboxSamplingOptions),
+    bottom: generateArray(letterboxSamplingOptions),
+  };
+}
+
+export function resetGradientSamples(samples: AardGradientSamples) {
+  for (let i = 0; i < samples.top.length; i++) {
+    for (let j = 0; j < samples.top[i].length; j++) {
+      samples.top[i][j] = 0;
+    }
+  }
+  for (let i = 0; i < samples.bottom.length; i++) {
+    for (let j = 0; j < samples.bottom[i].length; j++) {
+      samples.top[i][j] = 0;
+    }
+  }
+  if (samples.left) {
+    for (let i = 0; i < samples.left.length, i++) {
+      for (let j = 0; j < samples.left[i].length; j++) {
+        samples.left[i][j] = 0;
+      }
+    }
+  }
+  if (samples.right) {
+    for (let i = 0; i < samples.right.length; i++) {
+      for (let j = 0; j < samples.right[i].length; j++) {
+        samples.right[i][j] = 0;
+      }
+    }
+  }
+}
+

src/ext/lib/aard/interfaces/aard-test-results.interface.ts

@@ -1,3 +1,5 @@
+import { AardSettings } from '../../../../common/interfaces/SettingsInterface'
+
 export interface AardTestResults {
   isFinished: boolean,
   lastStage: number,
@@ -14,15 +16,29 @@ export interface AardTestResults {
     top: number,            // is cumulative
     bottom: number,         // is cumulative
     invalidated: boolean
-  }
+  },
+  aspectRatioCheck: {
+    topRows: [number, number, number],
+    topQuality: [number, number, number],
+    bottomRows: [number, number, number],
+    bottomQuality: [number, number, number],
+    topCandidate: number,
+    topCandidateQuality: number,
+    bottomCandidate: number,
+    bottomCandidateQuality: number,
+  },
+  aspectRatioUncertain: boolean,
+  letterboxWidth: number,
+  letterboxOffset: number,
+  logoDetected: [boolean, boolean, boolean, boolean]
 }
 
-export function initAardTestResults(): AardTestResults {
+export function initAardTestResults(settings: AardSettings): AardTestResults {
   return {
     isFinished: true,
     lastStage: 0,
     notLetterbox: false,
-    blackLevel: 0,
+    blackLevel: settings.blackLevels.defaultBlack,
     blackThreshold: 16,
     guardLine: {
       top: -1,
@@ -34,6 +50,31 @@ export function initAardTestResults(): AardTestResults {
       top: -1,
       bottom: -1,
       invalidated: false,
-    }
+    },
+    aspectRatioCheck: {
+      topRows: [-1, -1, -1],
+      topQuality: [0, 0, 0],
+      bottomRows: [-1, -1, -1],
+      bottomQuality: [0, 0, 0],
+      topCandidate: 0,
+      topCandidateQuality: 0,
+      bottomCandidate: 0,
+      bottomCandidateQuality: 0,
+    },
+    letterboxWidth: 0,
+    letterboxOffset: 0,
+    logoDetected: [false, false, false, false]
+
   }
 }
+
+export function resetAardTestResults(results: AardTestResults): void {
+  results.isFinished = false;
+  results.lastStage = 0;
+  results.notLetterbox = false;
+  results.guardLine.invalidated = false
+  results.guardLine.cornerViolations[0] = false;
+  results.guardLine.cornerViolations[1] = false;
+  results.guardLine.cornerViolations[2] = false;
+  results.guardLine.cornerViolations[3] = false;
+}