Minor fixes for comms and mutation detectìon

Todo: don't trip on subtitles that wouldn't get cropped, but that would require Aard to know player's aspect ratio, and it currently doesnt

src/ext/module/EventBus.ts

@@ -90,7 +90,24 @@ export default class EventBus {
     }
   }
 
+  private cloneContext(context: EventBusContext = {}): EventBusContext {
+    return {
+      stopPropagation: context.stopPropagation,
+      origin: context.origin,
+      frameUrl: context.frameUrl,
+      visitedBusses: context.visitedBusses ? [...context.visitedBusses] : undefined,
+      commandId: context.commandId,
+      comms: context.comms ? {
+        forwardTo: context.comms.forwardTo,
+        sourceFrame: context.comms?.sourceFrame ? { ...context.comms?.sourceFrame } : undefined
+      } : undefined,
+      borderCrossings: context.borderCrossings ? { ...context.borderCrossings } : undefined
+    };
+  }
+
   send(command: string, commandData: any, context: EventBusContext = {}) {
+    // context = this.cloneContext(context);  // Firefox throws an error if we don't clone the context.
+
     if (context.visitedBusses?.includes(this.uuid)) {
       console.warn('this bus was already visited before. Doing nothing.');
       return;
@@ -99,7 +116,10 @@ export default class EventBus {
       console.warn('this command was already sent');
       return;
     }
-    context.visitedBusses = [...context.visitedBusses ?? [], this.uuid];
+
+    // we want to avoid re-assigning context.visitedBusses if possible
+    // in order to reduce the amount of garbage that needs to be collected.
+    context.visitedBusses ? context.visitedBusses.push(this.uuid) : context.visitedBusses = [this.uuid];
 
     // execute commands we have subscriptions for
     if (this.commands?.[command]) {

src/ext/module/aard/Aard.ts

@@ -530,16 +530,26 @@ export class Aard {
           // console.warn('DETECTED NOT LETTERBOX! (resetting)')
           this.timer.arChanged();
           this.updateAspectRatio(this.defaultAr, {forceReset: true});
+          this.testResults.activeLetterbox.width = 0;
+          this.testResults.activeLetterbox.offset = 0;
+          this.testResults.activeLetterbox.orientation = LetterboxOrientation.NotLetterbox;
           break processUpdate;
         }
 
         if (this.testResults.subtitleDetected && arConf.subtitles.subtitleCropMode !== AardSubtitleCropMode.CropSubtitles) {
           if (arConf.subtitles.subtitleCropMode === AardSubtitleCropMode.ResetAR) {
             this.updateAspectRatio(this.defaultAr, {forceReset: true});
+            this.testResults.activeLetterbox.width = 0;
+            this.testResults.activeLetterbox.offset = 0;
+            this.testResults.activeLetterbox.orientation = LetterboxOrientation.NotLetterbox;
             this.timers.pauseUntil = Date.now() + arConf.subtitles.resumeAfter;
 
           } else if (arConf.subtitles.subtitleCropMode === AardSubtitleCropMode.ResetAndDisable) {
             this.updateAspectRatio(this.defaultAr, {forceReset: true});
+            this.testResults.activeLetterbox.width = 0;
+            this.testResults.activeLetterbox.offset = 0;
+            this.testResults.activeLetterbox.orientation = LetterboxOrientation.NotLetterbox;
+
             this.status.autoDisabled = true;
           }
 
@@ -568,10 +578,13 @@ export class Aard {
         this.testResults.guardLine.front = this.testResults.aspectRatioCheck.frontCandidate;
         this.testResults.guardLine.back = this.testResults.aspectRatioCheck.backCandidate;
 
-
+        // TODO: set flag if subtitles are far enough from edge to avoid getting cropped
         const finalAr = this.getAr();
         if (finalAr > 0) {
           this.updateAspectRatio(finalAr);
+          this.testResults.activeLetterbox.width = this.testResults.letterboxSize;
+          this.testResults.activeLetterbox.offset = this.testResults.letterboxOffset;
+          this.testResults.activeLetterbox.orientation = this.testResults.letterboxOrientation;
         } else {
           this.testResults.aspectRatioInvalid = true;
           this.testResults.aspectRatioInvalidReason = finalAr.toFixed(3);
@@ -843,6 +856,15 @@ export class Aard {
    * @returns
    */
   private updateLetterboxEdgeCandidates(crossDimension: number, topCandidate: number, bottomCandidate: number) {
+    // if new topCandidate or bottomCandidate aren't valid,
+    // use existing value.
+    if (topCandidate < 0) {
+      topCandidate = this.testResults.aspectRatioCheck.frontCandidate;
+    }
+    if (bottomCandidate < 0) {
+      bottomCandidate = this.testResults.aspectRatioCheck.backCandidate;
+    }
+
     const bottomDistance = (crossDimension - bottomCandidate);
     const maxOffset = ~~(crossDimension * this.settings.active.aard.edgeDetection.maxLetterboxOffset);
     const diff = Math.abs(topCandidate - bottomDistance);
@@ -858,6 +880,13 @@ export class Aard {
 
     this.testResults.letterboxSize = candidateAvg;
     this.testResults.letterboxOffset = diff;
+
+    if (this.testResults.letterboxOrientation === LetterboxOrientation.Letterbox && this.testResults.subtitleDetected) {
+      const top = this.testResults.subtitleScan.regions.top.firstSubtitle === -1 ? topCandidate : Math.min(topCandidate, this.testResults.subtitleScan.regions.top.firstSubtitle);
+      const bottom = Math.max(bottomCandidate, this.testResults.subtitleScan.regions.bottom.firstSubtitle);
+
+      this.testResults.letterboxSizeWithSubtitles = ~~((top + (crossDimension - bottomCandidate)) * 0.5);
+    }
   }
 
 
@@ -899,16 +928,36 @@ export class Aard {
     } else {
       this.testResults.aspectRatioUncertain = false;
     }
-    if (ssrRegions.top.firstSubtitle !== -1 || ssrRegions.bottom.firstSubtitle !== -1) {
-      this.testResults.subtitleDetected = true;
-    }
 
+    // 1. updateLetterboxEdgeCandidates runs regardless of whether we detected subtitles or not.
+    // 2. it's also not affected by whether subtitleDetected is set
+    // 3. we actually need to only reset letterbox if subtitle is actually outside of the crop area
     this.updateLetterboxEdgeCandidates(
       height,
       ssrRegions.top.firstImage,
       ssrRegions.bottom.firstImage
     );
 
+
+    // we can only do this in actual letterbox
+    if (this.testResults.activeLetterbox.orientation === LetterboxOrientation.Letterbox) {
+
+      // we only reset letterbox if letters are outside the video area, otherwise we risk
+      // getting whacked by credits, ppt youtubers, and other shit like that
+      const borderTop = this.testResults.activeLetterbox.width;
+      const borderBottom = this.settings.active.aard.canvasDimensions.sampleCanvas.height - this.testResults.activeLetterbox.width;
+
+      if (
+        (ssrRegions.top.firstSubtitle !== -1 && ssrRegions.top.firstSubtitle < borderTop)
+        || (ssrRegions.bottom.firstSubtitle !== -1 && ssrRegions.bottom.firstSubtitle > borderBottom)
+      ) {
+
+        this.testResults.subtitleDetected = true;
+      }
+
+    }
+
+
     this.timer.current.subtitleScan = performance.now() - this.timer.current.start;
   }
 
@@ -1370,9 +1419,17 @@ export class Aard {
     if (this.testResults.letterboxOrientation === LetterboxOrientation.Pillarbox) {
       const compensationFactor = compensatedWidth / this.canvasStore.main.width;
       const pillarboxCompensated = (this.testResults.letterboxSize * 2 * compensationFactor);
+
       return (compensatedWidth - pillarboxCompensated) / this.canvasStore.main.height;
     } else {
       const heightWithoutLetterbox = this.canvasStore.main.height - (this.testResults.letterboxSize * 2);
+
+      // TODO: set flag if subtitles are far enough from edge to avoid getting cropped
+      // if (this.testResults.subtitleDetected) {
+      //   const hwlWithSubtitles = this.canvasStore.main.height - (this.testResults.letterboxSizeWithSubtitles * 2)
+      //   const subtitleRatio = compensatedWidth / hwlWithSubtitles;
+
+      // }
       return compensatedWidth / heightWithoutLetterbox;
     }
   }

src/ext/module/aard/interfaces/aard-test-results.interface.ts

@@ -33,6 +33,7 @@ export interface AardTestResults {
   activeAspectRatio: number,  // is cumulative
   letterboxSize: number,
   letterboxOffset: number,
+  letterboxSizeWithSubtitles: number,
   aspectRatioInvalid: boolean,
   subtitleScan: {
     top: number,
@@ -43,6 +44,11 @@ export interface AardTestResults {
       bottom: AardTestResult_SubtitleRegion
     }
   },
+  activeLetterbox: {
+    width: number,
+    offset: number,
+    orientation: LetterboxOrientation
+  }
   aspectRatioUncertainReason?: AardUncertainReason,
   aspectRatioInvalidReason?: string,
 }
@@ -90,10 +96,16 @@ export function initAardTestResults(settings: AardSettings): AardTestResults {
         }
       }
     },
+    activeLetterbox: {
+      width: 0,
+      offset: 0,
+      orientation: LetterboxOrientation.NotLetterbox
+    },
     aspectRatioUpdated: false,
     activeAspectRatio: 0,
     letterboxSize: 0,
     letterboxOffset: 0,
+    letterboxSizeWithSubtitles: 0,
     aspectRatioInvalid: false,
   }
 }
@@ -107,7 +119,7 @@ export function resetAardTestResults(results: AardTestResults): void {
   results.isFinished = false;
   results.lastStage = 0;
   results.aspectRatioUpdated = false;
-  results.aspectRatioUncertainReason = null;
+  results.aspectRatioUncertainReason = undefined;
   results.aspectRatioInvalid = false;
   results.letterboxOrientation = LetterboxOrientation.NotKnown;
 }

src/ext/module/comms/CommsClient.ts

@@ -149,7 +149,11 @@ class CommsClient {
 
     // send to server
     if (!context?.borderCrossings?.commsServer) {
+      try {
         return chrome?.runtime?.sendMessage(null, message, null);
+      } catch (e) {
+        console.warn(`Failed to send message to background script. Error:`, e, 'data:', {message, context});
+      }
     }
   }
 

src/ext/module/comms/CommsServer.ts

@@ -266,11 +266,11 @@ class CommsServer {
    * @param frame
    */
   private async sendToOtherFrames(message, context) {
-    const sender = context.comms.sourceFrame;
+    const sender = context.comms?.sourceFrame;
 
     const enrichedMessage = {
       message,
-      _sourceFrame: context.comms.sourceFrame,
+      _sourceFrame: context.comms?.sourceFrame,
       _sourcePort: context.comms.port
     }
 

src/ext/module/video-data/IframeManager.ts

@@ -99,14 +99,14 @@ export default class IframeManager {
     } else {
       this.iframeList.push({
         ...data,
-        ...context.comms.sourceFrame
+        ...context.comms?.sourceFrame
       });
     }
   }
 
   private handleFrameDestroyed(context) {
     // tab IDs should be the same for all items, making frameId sufficiently unique to filter stuff
-    this.iframeList = this.iframeList.filter(x => x.frameId !== context.comms.sourceFrame.frameId);
+    this.iframeList = this.iframeList.filter(x => x.frameId !== context.comms?.sourceFrame?.frameId);
   }
 
 }

src/ext/module/video-data/VideoData.ts

@@ -47,6 +47,7 @@ class VideoData {
   videoLoaded: boolean = false;
   videoDimensionsLoaded: boolean = false;
   active: boolean = false;
+  private preventVideoOffsetValidation: boolean = false;
   //#endregion
 
   //#region misc stuff
@@ -511,6 +512,10 @@ class VideoData {
     if (this.destroyed) {
       return;
     }
+    if (!mutationList) {
+      this.logger.warn('onVideoMutation', 'mutation was triggered, but mutationList is missing. Something is fishy. Mutation will be ignored. Observer:', observer);
+      return;
+    }
 
     // verify that mutation didn't remove our class. Some pages like to do that.
     let confirmAspectRatioRestore = false;
@@ -530,7 +535,7 @@ class VideoData {
       return;
     }
 
-    for(const mutation of mutationList) {
+    for (const mutation of mutationList) {
       if (mutation.type === 'attributes') {
         if( mutation.attributeName === 'class'
             && mutation.oldValue.indexOf(this.baseCssName) !== -1
@@ -590,7 +595,9 @@ class VideoData {
 
     // sometimes something fucky wucky happens and mutations aren't detected correctly, so we
     // try to get around that
+    this.preventVideoOffsetValidation = true;
     setTimeout( () => {
+      this.preventVideoOffsetValidation = false;
       this.validateVideoOffsets();
     }, 100);
   }
@@ -611,6 +618,9 @@ class VideoData {
   }
 
   validateVideoOffsets() {
+    if (this.preventVideoOffsetValidation) {
+      return;
+    }
     // validate if current video still exists. If not, we destroy current object
     try {
       if (! document.body.contains(this.video)) {