Refactor aard

.vscode/settings.json

@@ -52,6 +52,7 @@
     "shitiness",
     "smallcaps",
     "suboption",
+    "subscan",
     "tabitem",
     "tablist",
     "tamius",

CHANGELOG.md

@@ -2,10 +2,12 @@
 
 ## v6.0 (current major)
 
-### v3.4.0
+### v6.4.0
 * In-player UI now appears in full-screen even for websites that use top layer
 * Embedded sites now inherit settings of the parent frame by default
+  * Setting inheritance/overriding is not thoroughly tested and may be full of edge cases.
 * Added validation to custom aspect ratio entry menu. Corrected parsing of aspect ratios given in the X:Y format, even though aspect ratios should be ideally given as a single number.
+* Autodetection can now scan for subtitles.
 
 ### v6.3.0
 * Added zoom segment to in-player UI and popup. 

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "ultrawidify",
-  "version": "6.3.93",
+  "version": "6.3.94",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ultrawidify",
-  "version": "6.3.93",
+  "version": "6.3.94",
   "description": "Aspect ratio fixer for youtube and other sites, with automatic aspect ratio detection. Supports ultrawide and other ratios.",
   "author": "Tamius Han <tamius.han@gmail.com>",
   "scripts": {

src/common/interfaces/SettingsInterface.ts

@@ -1,5 +1,6 @@
 import { Action } from '../../../node_modules/vuex/types/index'
 import { AardPollingOptions } from '../../ext/lib/aard/enums/aard-polling-options.enum'
+import { AardSubtitleCropMode } from '../../ext/lib/aard/enums/aard-subtitle-crop-mode.enum'
 import AntiGradientMode from '../enums/AntiGradientMode.enum'
 import AspectRatioType from '../enums/AspectRatioType.enum'
 import CropModePersistence from '../enums/CropModePersistence.enum'
@@ -66,6 +67,28 @@ export interface CommandInterface {
 export type SettingsReloadComponent = 'PlayerData' | 'VideoData';
 export type SettingsReloadFlags = true | SettingsReloadComponent;
 
+export interface AardSubtitleScanOptions {
+    subtitleCropMode: AardSubtitleCropMode,
+    resumeAfter: number,        // resume after X ms of no subtitle
+    scanSpacing: number,        // repeat subtitle scan every _n_ lines
+    scanMargin: number,         // % of pixels (from edge to start area) that we skip while scanning.
+                                // While technically anything between 0 and 0.5 is valid, the value
+                                // should be somewhere between 0.1-0.3 in order for subtitle scan to
+                                // work properly.
+    maxValidLetter: number,     // if letter is longer than this, something's off.
+    maxValidImage: number,
+    subtitleSubpixelThresholdOn: number,
+    subtitleSubpixelThresholdOff: number,
+    minImageSegment
+
+    minDetections: number,
+    minImageLineDetections: number,
+
+
+    refiningScanSpacing: number,  // must be base-2
+    refiningScanInitialIterations: number,
+}
+
 export interface AardSettings {
   aardType: 'webgl' | 'legacy' | 'auto';
 
@@ -93,10 +116,7 @@ export interface AardSettings {
     tickrate: number,          // 1 tick every this many milliseconds
   },
 
-  subtitles: {
+  subtitles: AardSubtitleScanOptions,
-    resetIfDetected?: boolean,  // reset if subtitles are detected
-    resumeAfter?: number,       // resume after X ms of no subtitles
-  },
 
   autoDisable: {            // settings for automatically disabling the extension
     onFirstChange: boolean,
@@ -164,6 +184,12 @@ export interface AardSettings {
                                     // are now. (NOTE: keep this less than 1 in case we implement logo detection)
     edgeMismatchTolerancePx: number,// corners and center are considered equal if they differ by at most this many px
   },
+
+  letterboxOrientationScan: {
+    letterboxLimit: number,  // how many non-black pixels we can detect before ruling out letterbox
+    pillarboxLimit: number,  // how many non-black pixels we can detect before ruling out pillarbox
+  }
+
   pillarTest: {
     ignoreThinPillarsPx: number, // ignore pillars that are less than this many pixels thick.
     allowMisaligned: number   // left and right edge can vary this much (%)

src/csui/src/PlayerUiPanels/AutodetectionSettingsPanel.vue

@@ -146,6 +146,45 @@
             </div>
           </div>
 
+          <div class="settings-segment">
+            <div class="warning-area">
+              Subtitle detection is experimental to high heaven.
+            </div>
+
+            <div class="field">
+              <div class="label">Subtitle detection:</div>
+              <div class="select">
+                <select v-model="settings.active.arDetect.subtitles.subtitleCropMode" @change="settings.saveWithoutReload">
+                  <option :value="AardSubtitleCropMode.DisableScan">Do not detect subtitles</option>
+                  <option :value="AardSubtitleCropMode.ResetAR">Do not crop while subtitles are on screen</option>
+                  <option :value="AardSubtitleCropMode.ResetAndDisable">Stop autodetection if subtitles are detected</option>
+                  <option :value="AardSubtitleCropMode.CropSubtitles">Always crop subtitles</option>
+                </select>
+              </div>
+            </div>
+
+            <div v-if="settings.active.arDetect.subtitles.subtitleCropMode === AardSubtitleCropMode.ResetAR" class="field">
+              <div class="label">Wait before resuming detection:</div>
+              <div class="range-input">
+                <input
+                  type="range"
+                  :value="Math.log(settings.active.arDetect.subtitles.resumeAfter / 10)"
+                  @change="setSubtitleTimeout($event.target.value, 10)"
+                  min="2.3"
+                  max="9.3"
+                  step="0.01"
+                />
+                <input
+                  :value="settings.active.arDetect.subtitles.resumeAfter / 1000"
+                  @change="setSubtitleTimeout($event.target.value, 1000)"
+                  class="input"
+                  type="text"
+                >
+                <div class="unit">s</div>
+              </div>
+            </div>
+          </div>
+
           <div v-if="settings.active.ui.devMode" class="settings-segment">
             <p>
               <b>Debug options</b>
@@ -191,6 +230,7 @@ import CropModePersistence from '../../../common/enums/CropModePersistence.enum'
 import AlignmentOptionsControlComponent from './AlignmentOptionsControlComponent.vue';
 import JsonEditor from '@csui/src/components/JsonEditor';
 import {AardPollingOptions} from '@src/ext/lib/aard/enums/aard-polling-options.enum';
+import {AardSubtitleCropMode} from '@src/ext/lib/aard/enums/aard-subtitle-crop-mode.enum';
 
 export default {
   components: {
@@ -217,6 +257,7 @@ export default {
 
       // enums n stuff
       AardPollingOptions,
+      AardSubtitleCropMode
     }
   },
   computed: {
@@ -251,6 +292,10 @@ export default {
       this.settings.active.arDetect.autoDisable.ifNotChangedTimeout = Math.floor(Math.pow(Math.E, event)) * multiplier;
       this.settings.saveWithoutReload();
     },
+    setSubtitleTimeout(event, multiplier) {
+      this.settings.active.arDetect.subtitles.resumeAfter = Math.floor(Math.pow(Math.E, event)) * multiplier;
+      this.settings.saveWithoutReload();
+    },
     refreshGraph() {
        this.eventBus.sendToTunnel('get-aard-timing');
     },

src/csui/src/PlayerUiPanels/ChangelogPanel.vue

@@ -8,8 +8,9 @@
         <h2>6.4.0</h2>
         <ul>
           <li>In-player UI can now appear in full-screen for websites that put players into top layer</li>
-          <li>Embedded sites now inherit settings of the parent frame</li>
+          <li>Embedded sites now inherit settings of the parent frame. <small>However, this hasn't been tested for all edge cases and may contain bugs.</small></li>
           <li>Added validation to custom aspect ratio entry menu. Corrected parsing of aspect ratios given in the X:Y format, even though aspect ratios should be ideally given as a single number.</li>
+          <li>Autodetection can be set to stop after first aspect ratio detection, or after a period of no changes.</li>
         </ul>
 
       </div>

src/ext/conf/ExtConfPatches.ts

@@ -259,6 +259,12 @@ const ExtensionConfPatch = Object.freeze([
       userOptions.arDetect.subtitles = defaultOptions.arDetect.subtitles;
       userOptions.arDetect.autoDisable = defaultOptions.arDetect.autoDisable;
     }
+  },
+  {
+    forVersion: '6.3.94',
+    updateFn: (userOptions: SettingsInterface, defaultOptions: SettingsInterface) => {
+      userOptions.arDetect.letterboxOrientationScan = defaultOptions.arDetect.letterboxOrientationScan;
+    }
   }
 
 ]);

src/ext/conf/ExtensionConf.ts

@@ -11,6 +11,7 @@ import BrowserDetect from './BrowserDetect';
 import { Extension } from 'typescript';
 import EmbeddedContentSettingsOverridePolicy from '../../common/enums/EmbeddedContentSettingsOverridePolicy.enum';
 import { AardPollingOptions } from '../lib/aard/enums/aard-polling-options.enum';
+import { AardSubtitleCropMode } from '../lib/aard/enums/aard-subtitle-crop-mode.enum';
 
 if(Debug.debug)
   console.log("Loading: ExtensionConf.js");
@@ -28,9 +29,25 @@ const ExtensionConf: SettingsInterface = {
       runOnSmallVideos: AardPollingOptions.Reduced
     },
 
+    letterboxOrientationScan: {
+      letterboxLimit: 8,
+      pillarboxLimit: 8
+    },
+
     subtitles: {
-      resetIfDetected: true,
+      subtitleCropMode: AardSubtitleCropMode.DisableScan,
       resumeAfter: 5000,
+      scanSpacing: 5,
+      scanMargin: 0.25,
+      maxValidLetter: 24,
+      maxValidImage: 0.33,
+      subtitleSubpixelThresholdOff: 8,
+      subtitleSubpixelThresholdOn: 192,
+      minDetections: 8,
+      minImageLineDetections: 16,
+
+      refiningScanSpacing: 8,
+      refiningScanInitialIterations: 12,
     },
 
     earlyStopOptions: {

src/ext/lib/aard/AardDebugUi.ts

@@ -196,14 +196,7 @@ export class AardDebugUi {
       <h2 style="color: #fa6; margin-bottom: 1rem">Detailed performance analysis:</h2>
       <div style="width: 100%; display: flex; flex-direction: column">
         <div style="margin-bottom: 1rem;">
-          <span style="color: #fff">Raw times (cumulative; -1 = test was skipped):</span><br/>
+          ${this.generateRawTimes(this.aard.timer.average)}
-          draw        <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.current.draw.toFixed(2)}ms</span>
-          get data    <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.current.getImage.toFixed(2)}ms</span>
-          black lv.   <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.current.fastBlackLevel.toFixed(2)}ms</span>
-          guard/image <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.current.guardLine.toFixed(3)}ms</span>
-          edge        <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.current.edgeScan.toFixed(2)}ms</span>
-          gradient    <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.current.gradient.toFixed(3)}ms</span>
-          post        <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.current.scanResults.toFixed(2)}ms</span>
         </div>
         <div style="color: #fff">Stage times (not cumulative):</div>
         <div style="display: flex; flex-direction: row; width: 100%; height: 150px">
@@ -212,14 +205,7 @@ export class AardDebugUi {
         </div>
 
         <div style="margin-bottom: 1rem;">
-          <span style="color: #fff">Raw times (cumulative; -1 = test was skipped):</span><br/>
+          ${this.generateRawTimes(this.aard.timer.average)}
-          draw        <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.average.draw.toFixed(2)}ms</span>
-          get data    <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.average.getImage.toFixed(2)}ms</span>
-          black lv.   <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.average.fastBlackLevel.toFixed(2)}ms</span>
-          guard/image <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.average.guardLine.toFixed(3)}ms</span>
-          edge        <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.average.edgeScan.toFixed(2)}ms</span>
-          gradient    <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.average.gradient.toFixed(3)}ms</span>
-          post        <span style="color: #fa6; margin-right: 0.5rem;">${this.aard.timer.average.scanResults.toFixed(2)}ms</span>
         </div>
         <div style="color: #fff">Stage times (not cumulative):</div>
         <div style="display: flex; flex-direction: row; width: 100%; height: 150px">
@@ -227,6 +213,9 @@ export class AardDebugUi {
           <div style="flex-grow: 1;">${this.generateMiniGraphBar(this.aard.timer.average, true)}</div>
         </div>
 
+        <div style="margin-bottom: 1rem;">
+          ${this.generateRawTimes(this.aard.timer.average)}
+        </div>
         <div style="display: flex; flex-direction: row; width: 100%; height: 150px">
           <div style="width: 160px; text-align: right; padding-right: 4px;">Last change:</div>
           <div style="flex-grow: 1;">${this.generateMiniGraphBar(this.aard.timer.lastChange, true)}</div>
@@ -274,6 +263,20 @@ export class AardDebugUi {
     `;
   }
 
+  private generateRawTimes(timer) {
+    return `
+      <span style="color: #fff">Raw times (cumulative; -1 = test was skipped):</span><br/>
+      draw        <span style="color: #fa6; margin-right: 0.5rem;">${timer.draw.toFixed(2)}ms</span>
+      get data    <span style="color: #fa6; margin-right: 0.5rem;">${timer.getImage.toFixed(2)}ms</span>
+      black lv.   <span style="color: #fa6; margin-right: 0.5rem;">${timer.fastBlackLevel.toFixed(2)}ms</span>
+      guard/image <span style="color: #fa6; margin-right: 0.5rem;">${timer.guardLine.toFixed(3)}ms</span>
+      edge        <span style="color: #fa6; margin-right: 0.5rem;">${timer.edgeScan.toFixed(2)}ms</span>
+      gradient    <span style="color: #fa6; margin-right: 0.5rem;">${timer.gradient.toFixed(3)}ms</span>
+      post        <span style="color: #fa6; margin-right: 0.5rem;">${timer.scanResults.toFixed(2)}ms</span>
+      subtitle    <span style="color: #fa6; margin-right: 0.5rem;">${timer.subtitleScan.toFixed(2)}ms</span>
+    `;
+  }
+
   private generateMiniGraphBar(perf: AardPerformanceData, detailed: boolean = false) {
     if (!perf) {
       return `
@@ -310,8 +313,12 @@ export class AardDebugUi {
     const scanResults = Math.max(perf.scanResults - total, 0);
     total += scanResults;
 
+    const subtitleScanStart = scanResultsStart + scanResults;
+    const subtitleScan  = Math.max(perf.scanResults - total, 0);
+    total += subtitleScan;
+
     return `
-      <div style="width: 100%; position: relative; text-align: right;">
+      <div style="width: calc(100% - 2rem); position: relative; text-align: right;">
         ${detailed ? '' : `${total.toFixed()} ms`}
         <div style="position: absolute; top: 0; left: 0; width: ${total}%; background: #fff; height: 2px;"></div>
         ${detailed ? `<div style="position: absolute; top: 0; left: ${total}%; height: 100px; width: 1px; border-left: 1px dotted rgba(255,255,255,0.5)"></div> ` : ''}
@@ -339,6 +346,11 @@ export class AardDebugUi {
 
         ${this.getBarLabel(0, scanResults + scanResultsStart, 88, `total: ${total.toFixed(2)} ms`, detailed, 'color: #fff;')}
 
+        <div style="position: absolute; top: ${detailed ? '74px' : '2px'}; left: ${subtitleScanStart}%; min-width: 1px; width: ${subtitleScan}%; background: rgba(234, 204, 84, 1); height: 12px;"></div>
+        ${this.getBarLabel(scanResults, scanResultsStart, 74, `scan results processing: ${subtitleScan.toFixed(2)} ms`, detailed)}
+
+        ${this.getBarLabel(0, scanResults + scanResultsStart, 88, `total: ${total.toFixed(2)} ms`, detailed, 'color: #fff;')}
+
         <!-- 60/30 fps markers -->
         <div style="position: absolute; top: ${detailed ? '-12px' : '0'}; left: 16.666%; width: 1px; border-left: 1px dashed #4f9; height: ${detailed ? '112px' : '12px'}; padding-left: 2px; background-color: rgba(0,0,0,0.5); z-index: ${detailed ? '5' : '2'}000;">60fps</div>
         <div style="position: absolute; top: ${detailed ? '-12px' : '0'}; left: 33.333%; width: 1px; border-left: 1px dashed #ff0; height: ${detailed ? '112px' : '12px'}; padding-left: 2px; background-color: #000; z-index: ${detailed ? '5' : '2'}000;">30fps</div>
@@ -346,6 +358,7 @@ export class AardDebugUi {
     `;
   }
 
+  _lastAr: undefined;
   updateTestResults(testResults) {
     this.updatePerformanceResults();
 
@@ -356,17 +369,22 @@ export class AardDebugUi {
 
     const resultsDiv = document.getElementById('uw-aard-results');
 
+    const ar = testResults.activeAspectRatio.toFixed(3);
+
     let out = `
       LAST STAGE: ${testResults.lastStage}         | black level: ${testResults.blackLevel}, threshold: ${testResults.blackThreshold}
 
       -- ASPECT RATIO
-      Active: ${testResults.activeAspectRatio.toFixed(3)}, changed since last check? ${testResults.aspectRatioUpdated}               letterbox width: ${testResults.letterboxWidth} offset ${testResults.letterboxOffset}
+      Active: ${ar}, changed since last check? ${testResults.aspectRatioUpdated}               letterbox width: ${testResults.letterboxWidth} offset ${testResults.letterboxOffset}<br/>
+      <sup>(last: ${this._lastAr})</sup>
+
 
       image in black level probe (aka "not letterbox"): ${testResults.notLetterbox}
-
     `;
+    this._lastAr = ar;
+
     if (testResults.notLetterbox) {
-      resultsDiv.textContent = out;
+      resultsDiv.innerHTML = out;
       return;
     }
     out = `${out}
@@ -406,7 +424,7 @@ export class AardDebugUi {
       `}
     `;
 
-    resultsDiv.textContent = out;
+    resultsDiv.innerHTML = out;
   }
 
   private setOverlayVisibility() {

src/ext/lib/aard/AardTimers.ts

@@ -7,6 +7,7 @@ export interface AardPerformanceData {
   guardLine: number; // actually times both guard line and image line checks
   edgeScan: number; // includes validation step
   gradient: number;
+  subtitleScan: number;
   scanResults: number;
 }
 
@@ -41,7 +42,8 @@ export class AardTimer {
       guardLine: -1,
       edgeScan: -1,
       gradient: -1,
-      scanResults: -1
+      scanResults: -1,
+      subtitleScan: -1,
     }
   };
 
@@ -53,6 +55,7 @@ export class AardTimer {
     this.aardPerformanceDataBuffer[index].edgeScan = -1;
     this.aardPerformanceDataBuffer[index].gradient = -1;
     this.aardPerformanceDataBuffer[index].scanResults = -1;
+    this.aardPerformanceDataBuffer[index].subtitleScan = -1;
   }
 
   next() {
@@ -73,6 +76,7 @@ export class AardTimer {
     this.lastChange.edgeScan = this.current.edgeScan;
     this.lastChange.gradient = this.current.gradient;
     this.lastChange.scanResults = this.current.scanResults;
+    this.lastChange.subtitleScan = this.current.scanResults;
   }
 
   getAverage() {
@@ -108,5 +112,6 @@ export class AardTimer {
     this.average.edgeScan /= this.aardPerformanceDataBuffer.length;
     this.average.gradient /= this.aardPerformanceDataBuffer.length;
     this.average.scanResults /= this.aardPerformanceDataBuffer.length;
+    this.average.subtitleScan /= this.aardPerformanceDataBuffer.length;
   }
 }

src/ext/lib/aard/enums/aard-letterbox-uncertain-reason.enum.ts

@@ -0,0 +1,5 @@
+export enum AardUncertainReason {
+  InsufficientEdgeDetectionQuality = 1,
+  LetterboxNotCenteredEnough = 2,
+  TopAndBottomRowMismatch = 3,
+}

src/ext/lib/aard/enums/aard-subtitle-crop-mode.enum.ts

@@ -0,0 +1,6 @@
+export enum AardSubtitleCropMode {
+  DisableScan = 0,
+  ResetAR = 1,
+  ResetAndDisable = 2,
+  CropSubtitles = 3
+}

src/ext/lib/aard/enums/edge.enum.ts

@@ -0,0 +1,6 @@
+export enum Edge {
+  Top = 1,
+  Left = 2,
+  Bottom = 3,
+  Right = 4
+}

src/ext/lib/aard/enums/letterbox-orientation.enum.ts

@@ -0,0 +1,7 @@
+export enum LetterboxOrientation {
+  NotKnown = -1,
+  NotLetterbox = 0,
+  Letterbox = 1,
+  Pillarbox = 2,
+  Both = 3
+}

src/ext/lib/aard/gl/GlDebugCanvas.ts

@@ -25,6 +25,7 @@ precision mediump float;
 uniform sampler2D u_texture;
 // uniform sampler1D u_colorTexture; // Array of replacement colors
 uniform vec3 u_colors[16];
+uniform vec3 u_colors2[16];
 varying vec2 vTextureCoord;
 
 void main() {
@@ -57,6 +58,26 @@ void main() {
     else if (alphaIndex == 14) selectedColor = u_colors[14];
     else selectedColor = u_colors[15];
 
+    gl_FragColor = vec4(selectedColor, 1.0);
+  } else if (alphaIndex < 32) {
+    vec3 selectedColor;
+    if (alphaIndex == 0) selectedColor = u_colors[0];
+    else if (alphaIndex == 1)  selectedColor = u_colors2[1];
+    else if (alphaIndex == 2)  selectedColor = u_colors2[2];
+    else if (alphaIndex == 3)  selectedColor = u_colors2[3];
+    else if (alphaIndex == 4)  selectedColor = u_colors2[4];
+    else if (alphaIndex == 5)  selectedColor = u_colors2[5];
+    else if (alphaIndex == 6)  selectedColor = u_colors2[6];
+    else if (alphaIndex == 7)  selectedColor = u_colors2[7];
+    else if (alphaIndex == 8)  selectedColor = u_colors2[8];
+    else if (alphaIndex == 9)  selectedColor = u_colors2[9];
+    else if (alphaIndex == 10) selectedColor = u_colors2[10];
+    else if (alphaIndex == 11) selectedColor = u_colors2[11];
+    else if (alphaIndex == 12) selectedColor = u_colors2[12];
+    else if (alphaIndex == 13) selectedColor = u_colors2[13];
+    else if (alphaIndex == 14) selectedColor = u_colors2[14];
+    else selectedColor = u_colors2[15];
+
     gl_FragColor = vec4(selectedColor, 1.0);
   } else {                                                         // red channel only as fallback
     gl_FragColor = vec4(color.r, 0.0, 0.0, 1.0);
@@ -77,7 +98,11 @@ export enum GlDebugType {
   EdgeScanHit = 9,
   SlopeTestDarkOk = 10,
   SlopeTestDarkViolation = 11,
-
+  SubtitleThresholdOff = 12,
+  SubtitleThresholdOn = 13,
+  SubtitleThresholdNone = 14,
+  LetterboxOrientationScanImageDetection = 15,
+  LetterboxOrientationScanTrace = 16,
 }
 
 export class GlDebugCanvas extends GlCanvas {
@@ -95,11 +120,29 @@ export class GlDebugCanvas extends GlCanvas {
     0.4, 0.4, 1.0,     // 9 - edge scan hit
     0.2, 0.4, 0.6,     // 10 - slope test ok
     1.0, 0.0, 0.0,     // 11 - slope test fail
-    0.0, 0.0, 0.0,   // 12
+    0.3, 0.3, 0.1,     // 12 - first subtitle off pixel
-    0.0, 0.0, 0.0,
+    1.0, 0.5, 0.0,     // 13 - first subtitle on pixel
-    0.0, 0.0, 0.0,
+    1.0, 1.0, 0.0,     // 14 - pixel subtitle — no threshold
-    0.0, 0.0, 0.0,
+    1.0, 0.0, 0.0,     // 15 - letterbox scan image detection
   ];
+  private debugColors2 = [
+    0.4, 0.3, 0.2,      // 16 - letterbox scan image trace
+    0.0, 0.0, 0.0,      // 17
+    0.0, 0.0, 0.0,      // 18
+    0.0, 0.0, 0.0,      // 19
+    0.0, 0.0, 0.0,      // 20
+    0.0, 0.0, 0.0,      // 21
+    0.0, 0.0, 0.0,      // 22
+    0.0, 0.0, 0.0,      // 23
+    0.0, 0.0, 0.0,      // 24
+    0.0, 0.0, 0.0,      // 25
+    0.0, 0.0, 0.0,      // 26
+    0.0, 0.0, 0.0,      // 27
+    0.0, 0.0, 0.0,      // 28
+    0.0, 0.0, 0.0,      // 29
+    0.0, 0.0, 0.0,      // 30
+    0.0, 0.0, 0.0,      // 31
+  ]
 
   constructor (options: GlCanvasOptions) {
     super(options);
@@ -132,6 +175,7 @@ export class GlDebugCanvas extends GlCanvas {
   enableFx() {
     this.gl.useProgram(this.programInfo.program)
     this.gl.uniform3fv((this.programInfo.uniformLocations as any).debugColors, this.debugColors);
+    this.gl.uniform3fv((this.programInfo.uniformLocations as any).debugColors2, this.debugColors2);
   }
 
   drawBuffer(buffer: Uint8Array) {
@@ -142,6 +186,7 @@ export class GlDebugCanvas extends GlCanvas {
     super.initWebgl();
 
     (this.programInfo.uniformLocations as any).debugColors = this.gl.getUniformLocation(this.programInfo.program, 'u_colors');
+    (this.programInfo.uniformLocations as any).debugColors2 = this.gl.getUniformLocation(this.programInfo.program, 'u_colors2');
   }
 
   protected updateTextureBuffer(buffer: Uint8Array) {

src/ext/lib/aard/interfaces/aard-detection-sample.interface.ts

@@ -19,6 +19,9 @@ export interface AardDetectionSample {
   bottom?: Int16Array;
   left?: Int16Array;
   right?: Int16Array;
+
+  start?: Int16Array;
+  end?: Int16Array;
 }
 
 export function generateSampleArray(samples: number, width: number, topBottom: boolean = true) {

src/ext/lib/aard/interfaces/aard-test-results.interface.ts

@@ -1,9 +1,22 @@
 import { AardSettings } from '../../../../common/interfaces/SettingsInterface'
+import { AardUncertainReason } from '../enums/aard-letterbox-uncertain-reason.enum'
+import { LetterboxOrientation } from '../enums/letterbox-orientation.enum'
+
+export interface AardTestResult_SubtitleRegion {
+  firstBlank: number,
+  lastBlank: number,
+  firstSubtitle: number,
+  lastSubtitle: number,
+  firstImage: number,
+  lastImage: number
+}
 
 export interface AardTestResults {
   isFinished: boolean,
   lastStage: number,
-  notLetterbox: boolean,
+  letterboxOrientation: LetterboxOrientation,
+  lastValidLetterboxOrientation: LetterboxOrientation,
+  subtitleDetected: boolean,
   blackLevel: number,       // is cumulative
   blackThreshold: number,   // is cumulative
   guardLine: {
@@ -36,19 +49,30 @@ export interface AardTestResults {
   bottomRowUncertain: boolean,
   aspectRatioUpdated: boolean,
   activeAspectRatio: number,  // is cumulative
-  letterboxWidth: number,
+  letterboxSize: number,
   letterboxOffset: number,
   logoDetected: [boolean, boolean, boolean, boolean]
-  aspectRatioInvalid: boolean
+  aspectRatioInvalid: boolean,
-  aspectRatioUncertainReason?: string
+  subtitleScan: {
-  aspectRatioInvalidReason?: string
+    top: number,
+    bottom: number,
+
+    regions: {
+      top: AardTestResult_SubtitleRegion,
+      bottom: AardTestResult_SubtitleRegion
+    }
+  },
+  aspectRatioUncertainReason?: AardUncertainReason,
+  aspectRatioUncertainEdges: number,
+  aspectRatioInvalidReason?: string,
 }
 
 export function initAardTestResults(settings: AardSettings): AardTestResults {
   return {
     isFinished: true,
     lastStage: 0,
-    notLetterbox: false,
+    letterboxOrientation: LetterboxOrientation.NotKnown,
+    lastValidLetterboxOrientation: LetterboxOrientation.NotKnown,
     blackLevel: settings.blackLevels.defaultBlack,
     blackThreshold: 16,
     guardLine: {
@@ -77,11 +101,36 @@ export function initAardTestResults(settings: AardSettings): AardTestResults {
       bottomRowsDifferenceMatrix: [0, 0, 0],
     },
     aspectRatioUncertain: false,
+    aspectRatioUncertainEdges: 0,
     topRowUncertain: false,
     bottomRowUncertain: false,
+    subtitleDetected: false,
+    subtitleScan: {
+      top: -1,
+      bottom: -1,
+
+      regions: {
+        top: {
+          firstBlank: -1,
+          lastBlank: -1,
+          firstSubtitle: -1,
+          lastSubtitle: -1,
+          firstImage: -1,
+          lastImage: -1
+        },
+        bottom: {
+          firstBlank: -1,
+          lastBlank: -1,
+          firstSubtitle: -1,
+          lastSubtitle: -1,
+          firstImage: -1,
+          lastImage: -1
+        }
+      }
+    },
     aspectRatioUpdated: false,
     activeAspectRatio: 0,
-    letterboxWidth: 0,
+    letterboxSize: 0,
     letterboxOffset: 0,
     logoDetected: [false, false, false, false],
     aspectRatioInvalid: false,
@@ -106,7 +155,6 @@ export function resetGuardLine(results: AardTestResults) {
 export function resetAardTestResults(results: AardTestResults): void {
   results.isFinished = false;
   results.lastStage = 0;
-  results.notLetterbox = false;
   results.imageLine.invalidated = false;
   results.guardLine.invalidated = false;
   results.guardLine.cornerViolated[0] = false;
@@ -124,4 +172,25 @@ export function resetAardTestResults(results: AardTestResults): void {
   results.topRowUncertain = false;
   results.bottomRowUncertain = false;
   results.aspectRatioInvalid = false;
+  results.letterboxOrientation = LetterboxOrientation.NotKnown;
 }
+
+export function resetSubtitleScanResults(results: AardTestResults): void {
+  results.subtitleScan.top = -1;
+  results.subtitleScan.bottom = -1;
+
+  results.subtitleScan.regions.top.firstBlank = -1;
+  results.subtitleScan.regions.top.lastBlank = -1;
+  results.subtitleScan.regions.top.firstSubtitle = -1;
+  results.subtitleScan.regions.top.lastSubtitle = -1;
+  results.subtitleScan.regions.top.firstImage = -1;
+  results.subtitleScan.regions.top.lastImage = -1;
+
+  results.subtitleScan.regions.bottom.firstBlank = -1;
+  results.subtitleScan.regions.bottom.lastBlank = -1;
+  results.subtitleScan.regions.bottom.firstSubtitle = -1;
+  results.subtitleScan.regions.bottom.lastSubtitle = -1;
+  results.subtitleScan.regions.bottom.firstImage = -1;
+  results.subtitleScan.regions.bottom.lastImage = -1;
+}
+

src/ext/lib/aard/interfaces/aard-timers.interface.ts

@@ -2,6 +2,7 @@ export interface AardTimers {
   autoDisableAt: number | undefined;
   nextFrameCheckTime: number;
   reducedPollingNextCheckTime: number;
+  pauseUntil: number;
 }
 
 export function initAardTimers(): AardTimers {
@@ -9,5 +10,6 @@ export function initAardTimers(): AardTimers {
     nextFrameCheckTime: 0,
     reducedPollingNextCheckTime: 0,
     autoDisableAt: undefined,
+    pauseUntil: 0,
   };
 }

src/manifest.json

@@ -2,7 +2,7 @@
   "manifest_version": 3,
   "name": "Ultrawidify",
   "description": "Removes black bars on ultrawide videos and offers advanced options to fix aspect ratio.",
-  "version": "6.3.93",
+  "version": "6.3.94",
   "icons": {
     "32":"res/icons/uw-32.png",
     "64":"res/icons/uw-64.png"