Updated toys

src/ext/lib/aard/Aard.ts

@@ -6,6 +6,7 @@ import Logger from '../Logger';
 import Settings from '../Settings';
 import { SiteSettings } from '../settings/SiteSettings';
 import VideoData from '../video-data/VideoData';
+import { AardDebugUi } from './AardDebugUi';
 import { Corner } from './enums/corner.enum';
 import { VideoPlaybackState } from './enums/video-playback-state.enum';
 import { FallbackCanvas } from './gl/FallbackCanvas';
@@ -254,6 +255,8 @@ export class Aard {
 
 
   private forceFullRecheck: boolean = true;
+
+  private debugConfig: any = {};
   //#endregion
 
   //#region getters
@@ -316,7 +319,6 @@ export class Aard {
 
     try {
       this.showDebugCanvas();
-      this.canvasStore.main.showCanvas();
     } catch (e) {
       console.error('FALIED TO CREATE DEBUGG CANVAS', e);
     }
@@ -369,8 +371,15 @@ export class Aard {
     if (!this.canvasStore.debug) {
       this.canvasStore.debug = new GlDebugCanvas({...this.settings.active.arDetect.canvasDimensions.sampleCanvas, id: 'uw-debug-gl'});
     }
-    this.canvasStore.debug.show();
+    this.canvasStore.debug.enableFx();
-    this.canvasStore.debug.drawVideoFrame(this.canvasStore.main.canvas);
+    if (!this.debugConfig.debugUi) {
+      this.debugConfig.debugUi = new AardDebugUi(this);
+      this.debugConfig.debugUi.initContainer();
+      this.debugConfig.debugUi.attachCanvases(this.canvasStore.main.canvas, this.canvasStore.debug.canvas);
+
+      // if we don't draw a dummy frame from _real_ sources, we can't update buffer later
+      this.canvasStore.debug.drawVideoFrame(this.canvasStore.main.canvas);
+    }
   }
   //#endregion
 
@@ -594,44 +603,51 @@ export class Aard {
 
       // If forceFullRecheck is set, then 'not letterbox' should always force-reset the aspect ratio
       // (as aspect ratio may have been set manually while autodetection was off)
-      if (this.testResults.notLetterbox) {
-        // console.log('————not letterbox')
-        console.warn('DETECTED NOT LETTERBOX! (resetting)')
-        this.updateAspectRatio(this.defaultAr);
-        return;
-      }
 
-      // if detection is uncertain, we don't do anything at all (unless if guardline was broken, in which case we reset)
+      // If debugging is enable,
-      if (this.testResults.aspectRatioUncertain && this.testResults.guardLine.invalidated) {
+      this.canvasStore.debug?.drawBuffer(imageData);
-        // console.info('aspect ratio not certain:', this.testResults.aspectRatioUncertainReason);
-        // console.warn('check finished:', JSON.parse(JSON.stringify(this.testResults)), JSON.parse(JSON.stringify(this.canvasSamples)), '\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n');
 
-        console.warn('ASPECT RATIO UNCERTAIN, GUARD LINE INVALIDATED (resetting)')
+      do {
-        this.updateAspectRatio(this.defaultAr);
+        if (this.testResults.notLetterbox) {
+          // console.log('————not letterbox')
+          console.warn('DETECTED NOT LETTERBOX! (resetting)')
+          this.updateAspectRatio(this.defaultAr);
+          break;
+        }
 
-        return;
+        // if detection is uncertain, we don't do anything at all (unless if guardline was broken, in which case we reset)
-      }
+        if (this.testResults.aspectRatioUncertain && this.testResults.guardLine.invalidated) {
+          // console.info('aspect ratio not certain:', this.testResults.aspectRatioUncertainReason);
+          // console.warn('check finished:', JSON.parse(JSON.stringify(this.testResults)), JSON.parse(JSON.stringify(this.canvasSamples)), '\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n');
 
-      // TODO: emit debug values if debugging is enabled
+          console.warn('ASPECT RATIO UNCERTAIN, GUARD LINE INVALIDATED (resetting)')
-      this.testResults.isFinished = true;
+          this.updateAspectRatio(this.defaultAr);
 
-      console.warn(
+          break;
-        `[${(+new Date() % 10000) / 100} | ${this.arid}]`,'check finished — aspect ratio updated:', this.testResults.aspectRatioUpdated,
+        }
-        '\ndetected ar:', this.testResults.activeAspectRatio, '->', this.getAr(),
-        '\nis video playing?', this.getVideoPlaybackState() === VideoPlaybackState.Playing,
-         '\n\n', JSON.parse(JSON.stringify(this.testResults)), JSON.parse(JSON.stringify(this.canvasSamples)), '\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n');
 
-      // if edge width changed, emit update event.
+        // TODO: emit debug values if debugging is enabled
-      // except aspectRatioUpdated doesn't get set reliably, so we just call update every time, and update
+        this.testResults.isFinished = true;
-      // if detected aspect ratio is different from the current aspect ratio
-      // if (this.testResults.aspectRatioUpdated) {
-        this.updateAspectRatio();
-      // }
 
-      // if we got "no letterbox" OR aspectRatioUpdated
+        console.warn(
+          `[${(+new Date() % 10000) / 100} | ${this.arid}]`,'check finished — aspect ratio updated:', this.testResults.aspectRatioUpdated,
+          '\ndetected ar:', this.testResults.activeAspectRatio, '->', this.getAr(),
+          '\nis video playing?', this.getVideoPlaybackState() === VideoPlaybackState.Playing,
+          '\n\n', JSON.parse(JSON.stringify(this.testResults)), JSON.parse(JSON.stringify(this.canvasSamples)), '\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n');
+
+        // if edge width changed, emit update event.
+        // except aspectRatioUpdated doesn't get set reliably, so we just call update every time, and update
+        // if detected aspect ratio is different from the current aspect ratio
+        // if (this.testResults.aspectRatioUpdated) {
+          this.updateAspectRatio();
+        // }
+
+        // if we got "no letterbox" OR aspectRatioUpdated
+      } while (false)
 
       if (this.canvasStore.debug) {
-        this.canvasStore.debug.drawBuffer(imageData);
+        // this.canvasStore.debug.drawBuffer(imageData);
+        this.debugConfig?.debugUi?.updateTestResults(this.testResults);
       }
     } catch (e) {
       console.warn('[Ultrawidify] Aspect ratio autodetection crashed for some reason.\n\nsome reason:', e);

src/ext/lib/aard/AardDebugUi.ts

@@ -0,0 +1,177 @@
+import { Aard } from './../../../../dist-chrome/ext/lib/aard/Aard';
+
+
+export class AardDebugUi {
+
+  aard: any;
+
+  uiAnchorElement: HTMLDivElement;
+  pauseOnArCheck: boolean = false;
+
+  constructor(aard: any) {
+    this.aard = aard;
+
+    (window as any).ultrawidify_uw_aard_debug_tools = {
+      enableStopOnChange: () => this.changePauseOnCheck(true),
+      disableStopOnChange: () => this.changePauseOnCheck(false),
+      resumeVideo: () => this.resumeVideo(),
+      step: () => this.aard.step()
+    }
+  }
+
+  initContainer() {
+    const div = document.createElement('div');
+    div.id = 'uw-aard-debug-ui-container';
+    div.innerHTML = `
+     <div style="
+        position: fixed; top: 0; left: 0; width: 100vw; height: 100dvh; pointer-events: none; z-index: 9999; display: flex; flex-direction: row; font-size: 16px; font-family: 'Overpass Mono', monospace;
+      ">
+        <div style="">
+          <div id="uw-aard-debug_aard-sample-canvas" style="min-width: 640px"></div>
+          <div style="background: black; color: #fff"; font-size: 24px;">AARD IN</div>
+
+          <div style="background: black; color: #ccc; padding: 1rem">
+            <div>
+              <span style="color: rgb(0.1, 0.1, 0.35)">■</span>
+              Black level sample
+            </div>
+            <div>
+              <span style="color: rgb(0.3, 1.0, 0.6)">■</span>
+              <span style="color: rgb(0.1, 0.5, 0.3)">■</span>
+              Guard line (middle/corner) OK
+            </div>
+            <div>
+              <span style="color: rgb(1.0, 0.1, 0.1)">■</span>
+              <span style="color: rgb(0.5, 0.0, 0.0)">■</span>
+              Guard line (middle/corner) violation
+            </div>
+            <div>
+              Image line — <span style="color: rgb(0.7, 0.7, 0.7)">■</span> image, <span style="color: rgb(0.2, 0.2, 0.6)">■</span> no image
+            </div>
+            <div>
+              Edge scan — <span style="color: rgb(0.1, 0.1, 0.4)">■</span> probe, <span style="color: rgb(0.4, 0.4, 1.0)">■</span> hit
+            </div>
+            <div>
+              Slope test — <span style="color: rgb(0.4, 0.4, 1.0)">■</span> ok, <span style="color: rgb(1.0, 0.0, 0.0)">■</span> fail
+            </div>
+          </div>
+
+          <div style="pointer-events: all">
+            <button id="uw-aard-debug-ui_enable-stop-on-change"  style="">Pause video on aspect ratio change</button>
+            <button id="uw-aard-debug-ui_disable-stop-on-change" style="display: none">Stop pausing video on aspect ratio change</button>
+            <button id="uw-aard-debug-ui_resume-video"           >Resume video</button>
+            <button id="uw-aard-debug-ui_enable-step"            >Run aspect ratio detection</button>
+          </div>
+        </div>
+
+        <div style="flex-grow: 1"></div>
+        <div>
+          <div style="background: black; color: #ccc;">
+            <div style="font-size: 24px; padding: 1rem;">
+              Debug results:
+            </div>
+            <pre id="uw-aard-results"></pre>
+          </div>
+        </div>
+        <div style="width: 1920px">
+          <div id="uw-aard-debug_aard-output" style="zoom: 3; image-rendering: pixelated;"></div>
+          <div style="background: black; color: #fff; font-size: 24px;">AARD RESULT</div>
+
+        </div>
+      </div>
+    `;
+
+    document.body.appendChild(div);
+    this.uiAnchorElement = div;
+
+    document.getElementById('uw-aard-debug-ui_enable-stop-on-change').onclick = () => this.changePauseOnCheck(true);
+    document.getElementById('uw-aard-debug-ui_disable-stop-on-change').onclick = () => this.changePauseOnCheck(true);
+    document.getElementById('uw-aard-debug-ui_resume-video').onclick = () => this.resumeVideo();
+    document.getElementById('uw-aard-debug-ui_enable-step').onclick = () => this.aard.step();
+  }
+
+  changePauseOnCheck(pauseOnChange: boolean) {
+    this.pauseOnArCheck = pauseOnChange;
+
+    document.getElementById("uw-aard-debug-ui_enable-stop-on-change").style.display = pauseOnChange ? "none" : "";
+    document.getElementById("uw-aard-debug-ui_disable-stop-on-change").style.display = pauseOnChange ? "" : "none";
+  }
+
+  destroyContainer() {
+    this.uiAnchorElement.remove();
+  }
+
+  attachCanvases(sample: HTMLCanvasElement, debug: HTMLCanvasElement) {
+    const sampleCanvasParent = document.getElementById('uw-aard-debug_aard-sample-canvas');
+    sampleCanvasParent.appendChild(sample);
+    const debugCanvasParent = document.getElementById('uw-aard-debug_aard-output');
+    debugCanvasParent.appendChild(debug);
+  }
+
+  resumeVideo() {
+    (this.aard as any).video.play();
+    this.aard.start();
+  }
+
+  updateTestResults(testResults) {
+    if (testResults.aspectRatioUpdated && this.pauseOnArCheck) {
+      (this.aard as any).video.pause();
+      this.aard.stop();
+    }
+
+    const resultsDiv = document.getElementById('uw-aard-results');
+
+    let out = `
+      LAST STAGE: ${testResults.lastStage}         | black level: ${testResults.blackLevel}, threshold: ${testResults.blackThreshold}
+
+      -- ASPECT RATIO
+      Active: ${testResults.activeAspectRatio.toFixed(3)}, changed since last check? ${testResults.aspectRatioUpdated}               letterbox width: ${testResults.letterboxWidth} offset ${testResults.letterboxOffset}
+
+      image in black level probe (aka "not letterbox"): ${testResults.notLetterbox}
+
+    `;
+    if (testResults.notLetterbox) {
+      resultsDiv.textContent = out;
+      return;
+    }
+    out = `${out}
+
+      -- UNCERTAIN FLAGS
+      AR: ${testResults.aspectRatioUncertain} (reason: ${testResults.aspectRatioUncertainReason ?? 'n/a'}); top row: ${testResults.topRowUncertain}; bottom row: ${testResults.bottomRowUncertain}
+
+      -- GUARD & IMAGE LINE
+      bottom guard: ${testResults.guardLine.bottom}       image: ${testResults.guardLine.invalidated ? 'n/a' : testResults.imageLine.bottom}
+      top    guard: ${testResults.guardLine.top}      image: ${testResults.guardLine.invalidated ? 'n/a' : testResults.imageLine.top}
+
+      guard line ${testResults.guardLine.invalidated ? 'INVALIDATED' : 'valid'}       image line ${testResults.guardLine.invalidated ? '<skipped test>' : testResults.imageLine.invalidated ? 'INVALIDATED' : 'valid'}
+
+      corner invalidations (invalid pixels -> verdict)
+
+                    LEFT         CENTER         RIGHT
+      bottom:       ${testResults.guardLine.cornerPixelsViolated[0]} → ${testResults.guardLine.cornerViolated[0] ? '❌' : '◽'}                       ${testResults.guardLine.cornerPixelsViolated[1]} → ${testResults.guardLine.cornerViolated[1] ? '❌' : '◽'}
+         top:       ${testResults.guardLine.cornerPixelsViolated[2]} → ${testResults.guardLine.cornerViolated[2] ? '❌' : '◽'}                       ${testResults.guardLine.cornerPixelsViolated[3]} → ${testResults.guardLine.cornerViolated[3] ? '❌' : '◽'}
+
+      -- AR SCAN ${testResults.lastStage < 1 ? `
+      DID NOT RUN THIS FRAME` : `
+
+                     LEFT         CENTER         RIGHT        CANDIDATE
+      BOTTOM
+      distance:       ${testResults.aspectRatioCheck.bottomRows[0]}             ${testResults.aspectRatioCheck.bottomRows[1]}             ${testResults.aspectRatioCheck.bottomRows[2]}             ${testResults.aspectRatioCheck.bottomCandidate}
+       quality:       ${testResults.aspectRatioCheck.bottomQuality[0]}              ${testResults.aspectRatioCheck.bottomQuality[1]}              ${testResults.aspectRatioCheck.bottomQuality[2]}              ${testResults.aspectRatioCheck.bottomCandidateQuality}
+
+      TOP
+      distance:       ${testResults.aspectRatioCheck.topRows[0]}             ${testResults.aspectRatioCheck.topRows[1]}             ${testResults.aspectRatioCheck.topRows[2]}             ${testResults.aspectRatioCheck.topCandidate}
+       quality:       ${testResults.aspectRatioCheck.topQuality[0]}              ${testResults.aspectRatioCheck.topQuality[1]}              ${testResults.aspectRatioCheck.topQuality[2]}              ${testResults.aspectRatioCheck.topCandidateQuality}
+
+      Diff matrix:
+                  R-L      C-R      C-L
+        bottom:    ${testResults.aspectRatioCheck.bottomRowsDifferenceMatrix[0]}       ${testResults.aspectRatioCheck.bottomRowsDifferenceMatrix[1]}       ${testResults.aspectRatioCheck.bottomRowsDifferenceMatrix[2]}
+           top:    ${testResults.aspectRatioCheck.topRowsDifferenceMatrix[0]}       ${testResults.aspectRatioCheck.topRowsDifferenceMatrix[1]}       ${testResults.aspectRatioCheck.topRowsDifferenceMatrix[2]}
+      `}
+    `;
+
+    resultsDiv.textContent = out;
+  }
+
+}
+

src/ext/lib/aard/gl/GlDebugCanvas.ts

@@ -83,14 +83,14 @@ export enum GlDebugType {
 export class GlDebugCanvas extends GlCanvas {
 
   private debugColors = [
-    0.1, 0.1, 0.25,    // 0 - black level sample
+    0.1, 0.1, 0.35,    // 0 - black level sample
     0.3, 1.0, 0.6,     // 1 - guard line ok
     1.0, 0.1, 0.1,     // 2 - guard line violation
-    0.2, 0.6, 0.4,     // 3 - guard line corner ok
+    0.1, 0.5, 0.3,     // 3 - guard line corner ok
     0.5, 0.0, 0.0,     // 4 - guard line corner violation
     1.0, 1.0, 1.0,     // 5 - image line threshold reached (stop checking)
     0.7, 0.7, 0.7,     // 6 - image line ok
-    0.9, 0.6, 0.6,     // 7 - image line fail
+    0.2, 0.2, 0.6,     // 7 - image line fail
     0.1, 0.1, 0.4,     // 8 - edge scan probe
     0.4, 0.4, 1.0,     // 9 - edge scan hit
     0.2, 0.4, 0.6,     // 10 - slope test ok
@@ -114,20 +114,24 @@ export class GlDebugCanvas extends GlCanvas {
   }
 
   show() {
+    this.enableFx();
+
+    // this.canvas.style.position = 'fixed';
+    // this.canvas.style.top = '0';
+    // this.canvas.style.right = '0';
+    // this.canvas.style.zIndex = '99999999';
+    // this.canvas.style.transform = 'scale(3)';
+    // this.canvas.style.transformOrigin = 'top right';
+    // this.canvas.style.imageRendering = 'pixelated';
+
+    // document.body.appendChild(
+    //   this.canvas
+    // );
+  }
+
+  enableFx() {
     this.gl.useProgram(this.programInfo.program)
     this.gl.uniform3fv((this.programInfo.uniformLocations as any).debugColors, this.debugColors);
-
-    this.canvas.style.position = 'fixed';
-    this.canvas.style.top = '0';
-    this.canvas.style.right = '0';
-    this.canvas.style.zIndex = '99999999';
-    this.canvas.style.transform = 'scale(3)';
-    this.canvas.style.transformOrigin = 'top right';
-    this.canvas.style.imageRendering = 'pixelated';
-
-    document.body.appendChild(
-      this.canvas
-    );
   }
 
   drawBuffer(buffer: Uint8Array) {