Tamius Han
e269ff3d79
Todo: don't trip on subtitles that wouldn't get cropped, but that would require Aard to know player's aspect ratio, and it currently doesnt
1440 lines
52 KiB
TypeScript
1440 lines
52 KiB
TypeScript
import AspectRatioType from '@src/common/enums/AspectRatioType.enum';
|
||
import ExtensionMode from '@src/common/enums/ExtensionMode.enum';
|
||
import { ArVariant } from '@src/common/interfaces/ArInterface';
|
||
import { ExtensionEnvironment } from '@src/common/interfaces/SettingsInterface';
|
||
import EventBus from '../EventBus';
|
||
import Settings from '../settings/Settings';
|
||
import { SiteSettings } from '../settings/SiteSettings';
|
||
import VideoData from '../video-data/VideoData';
|
||
import { AardDebugUi } from './AardDebugUi';
|
||
import { AardTimer } from './AardTimers';
|
||
import { Corner } from './enums/corner.enum';
|
||
import { VideoPlaybackState } from './enums/video-playback-state.enum';
|
||
import { FallbackCanvas } from './gl/FallbackCanvas';
|
||
import { GlCanvas } from './gl/GlCanvas';
|
||
import { GlDebugCanvas, GlDebugType } from './gl/GlDebugCanvas';
|
||
import { AardCanvasStore } from './interfaces/aard-canvas-store.interface';
|
||
import { AardDetectionSample, generateSampleArray, resetSamples } from './interfaces/aard-detection-sample.interface';
|
||
import { AardStatus, initAardStatus } from './interfaces/aard-status.interface';
|
||
import { AardTestResult_SubtitleRegion, AardTestResults, initAardTestResults, resetAardTestResults, resetGuardLine, resetSubtitleScanResults } from './interfaces/aard-test-results.interface';
|
||
import { AardTimers, initAardTimers } from './interfaces/aard-timers.interface';
|
||
import { ComponentLogger } from '../logging/ComponentLogger';
|
||
import { AardPollingOptions } from './enums/aard-polling-options.enum';
|
||
import { AardSubtitleCropMode } from './enums/aard-subtitle-crop-mode.enum';
|
||
import { LetterboxOrientation } from './enums/letterbox-orientation.enum';
|
||
import { Edge } from './enums/edge.enum';
|
||
import { AardUncertainReason } from './enums/aard-letterbox-uncertain-reason.enum';
|
||
import { result } from 'lodash';
|
||
|
||
|
||
/**
|
||
* /\
|
||
* //\\ Automatic
|
||
* // \\ Aspect
|
||
* // \\ Ratio
|
||
* \\ Detector
|
||
* //XXXX \\
|
||
* // \\ (Totes not a Witcher reference)
|
||
* // \\ (Witcher 2 best Witcher)
|
||
* //XXXXXXXXXXXXXX\\
|
||
*
|
||
*/
|
||
|
||
const PIXEL_SIZE = 4;
|
||
const PIXEL_SIZE_FRACTION = 0.25;
|
||
let ROW_SIZE = -1;
|
||
|
||
export class Aard {
|
||
//#region configuration parameters
|
||
private logger: ComponentLogger;
|
||
private videoData: VideoData;
|
||
private settings: Settings;
|
||
private siteSettings: SiteSettings;
|
||
private eventBus: EventBus;
|
||
private arid: string;
|
||
private arVariant: ArVariant;
|
||
|
||
private eventBusCommands = {
|
||
'uw-environment-change': {
|
||
function: (newEnvironment: ExtensionEnvironment) => {
|
||
// console.log('received extension environment:', newEnvironment, 'player env:', this.videoData?.player?.environment);
|
||
this.startCheck();
|
||
}
|
||
},
|
||
'aard-enable-debug': {
|
||
function: (enabled: boolean) => {
|
||
if (enabled) {
|
||
this.showDebugCanvas();
|
||
} else {
|
||
this.hideDebugCanvas();
|
||
}
|
||
}
|
||
},
|
||
'url-changed': {
|
||
function: () => {
|
||
this.clearAutoDisabled();
|
||
}
|
||
}
|
||
// 'get-aard-timing': {
|
||
// function: () => this.handlePerformanceDataRequest()
|
||
// }
|
||
};
|
||
//#endregion
|
||
|
||
private video: HTMLVideoElement;
|
||
|
||
private animationFrame: number;
|
||
|
||
//#region internal state
|
||
public status: AardStatus = initAardStatus();
|
||
private timers: AardTimers = initAardTimers();
|
||
private inFallback: boolean = false;
|
||
private fallbackReason: any;
|
||
private canvasStore: AardCanvasStore;
|
||
private testResults: AardTestResults;
|
||
private verticalTestResults: AardTestResults;
|
||
private canvasSamples: AardDetectionSample;
|
||
|
||
|
||
private forceFullRecheck: boolean = true;
|
||
|
||
private debugConfig: any = {};
|
||
private timer: AardTimer;
|
||
private lastAnimationFrameTime: number = Infinity;
|
||
//#endregion
|
||
|
||
//#region getters
|
||
get defaultAr() {
|
||
if (!this.video) {
|
||
return undefined;
|
||
}
|
||
|
||
this.video.setAttribute('crossOrigin', 'anonymous');
|
||
|
||
const ratio = this.video.videoWidth / this.video.videoHeight;
|
||
if (isNaN(ratio)) {
|
||
return undefined;
|
||
}
|
||
return ratio;
|
||
}
|
||
|
||
//#endregion getters
|
||
|
||
//#region lifecycle
|
||
constructor(videoData: VideoData){
|
||
this.logger = new ComponentLogger(videoData.logAggregator, 'Aard', {});
|
||
this.videoData = videoData;
|
||
this.video = videoData.video;
|
||
this.settings = videoData.settings;
|
||
this.siteSettings = videoData.siteSettings;
|
||
this.eventBus = videoData.eventBus;
|
||
|
||
this.eventBus.subscribeMulti(this.eventBusCommands, this);
|
||
|
||
this.arid = (Math.random()*100).toFixed();
|
||
|
||
// we can tick manually, for debugging
|
||
this.logger.log('ctor', `creating new ArDetector. arid: ${this.arid}`);
|
||
|
||
this.timer = new AardTimer();
|
||
this.init();
|
||
}
|
||
|
||
/**
|
||
* Initializes Aard with default values and starts autodetection loop.
|
||
* This method should only ever be called from constructor.
|
||
*/
|
||
private init() {
|
||
this.canvasStore = {
|
||
main: this.createCanvas('main-gl')
|
||
};
|
||
|
||
|
||
this.canvasSamples = {
|
||
top: generateSampleArray(
|
||
this.settings.active.aard.sampling.staticCols,
|
||
this.settings.active.aard.canvasDimensions.sampleCanvas.width
|
||
),
|
||
bottom: generateSampleArray(
|
||
this.settings.active.aard.sampling.staticCols,
|
||
this.settings.active.aard.canvasDimensions.sampleCanvas.width
|
||
),
|
||
left: generateSampleArray(
|
||
this.settings.active.aard.sampling.staticCols,
|
||
this.settings.active.aard.canvasDimensions.sampleCanvas.height
|
||
),
|
||
right: generateSampleArray(
|
||
this.settings.active.aard.sampling.staticCols,
|
||
this.settings.active.aard.canvasDimensions.sampleCanvas.height
|
||
)
|
||
};
|
||
|
||
|
||
// try {
|
||
// this.showDebugCanvas();
|
||
// } catch (e) {
|
||
// console.error('FALIED TO CREATE DEBUGG CANVAS', e);
|
||
// }
|
||
|
||
try {
|
||
if (this.settings.active.ui.dev?.aardDebugOverlay?.showOnStartup) {
|
||
this.showDebugCanvas();
|
||
}
|
||
} catch (e) {
|
||
console.error(`[uw::aard] failed to create debug UI:`, e);
|
||
}
|
||
|
||
this.startCheck();
|
||
}
|
||
|
||
private createCanvas(canvasId: string, canvasType?: 'webgl' | 'legacy') {
|
||
ROW_SIZE = this.settings.active.aard.canvasDimensions.sampleCanvas.width * PIXEL_SIZE;
|
||
|
||
if (canvasType) {
|
||
if (canvasType === this.settings.active.aard.aardType || this.settings.active.aard.aardType === 'auto') {
|
||
if (canvasType === 'webgl') {
|
||
return new GlCanvas({...this.settings.active.aard.canvasDimensions.sampleCanvas, id: 'main-gl'});
|
||
} else if (canvasType === 'legacy') {
|
||
return new FallbackCanvas({...this.settings.active.aard.canvasDimensions.sampleCanvas, id: 'main-legacy'});
|
||
} else {
|
||
// TODO: throw error
|
||
}
|
||
} else {
|
||
// TODO: throw error
|
||
}
|
||
|
||
}
|
||
|
||
if (['auto', 'webgl'].includes(this.settings.active.aard.aardType)) {
|
||
try {
|
||
return new GlCanvas({...this.settings.active.aard.canvasDimensions.sampleCanvas, id: 'main-gl'});
|
||
} catch (e) {
|
||
if (this.settings.active.aard.aardType !== 'webgl') {
|
||
return new FallbackCanvas({...this.settings.active.aard.canvasDimensions.sampleCanvas, id: 'main-legacy'});
|
||
}
|
||
this.logger.error('createCanvas', 'could not create webgl canvas:', e);
|
||
this.eventBus.send('uw-config-broadcast', {type: 'aard-error', aardErrors: {webglError: true}});
|
||
throw e;
|
||
}
|
||
} else if (this.settings.active.aard.aardType === 'legacy') {
|
||
return new FallbackCanvas({...this.settings.active.aard.canvasDimensions.sampleCanvas, id: 'main-legacy'});
|
||
} else {
|
||
this.logger.error('createCanvas', 'invalid value in settings.arDetect.aardType:', this.settings.active.aard.aardType);
|
||
this.eventBus.send('uw-config-broadcast', {type: 'aard-error', aardErrors: {invalidSettings: true}});
|
||
throw 'AARD_INVALID_SETTINGS';
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Creates and shows debug canvas
|
||
* @param canvasId
|
||
*/
|
||
private showDebugCanvas() {
|
||
if (!this.canvasStore.debug) {
|
||
this.canvasStore.debug = new GlDebugCanvas({...this.settings.active.aard.canvasDimensions.sampleCanvas, id: 'uw-debug-gl'});
|
||
}
|
||
this.canvasStore.debug.enableFx();
|
||
if (!this.debugConfig.debugUi) {
|
||
this.debugConfig.debugUi = new AardDebugUi(this);
|
||
this.debugConfig.debugUi.initContainer();
|
||
this.debugConfig.debugUi.attachCanvases(this.canvasStore.main.canvas, this.canvasStore.debug.canvas);
|
||
|
||
// if we don't draw a dummy frame from _real_ sources, we can't update buffer later
|
||
this.canvasStore.debug.drawVideoFrame(this.canvasStore.main.canvas);
|
||
}
|
||
}
|
||
|
||
private hideDebugCanvas() {
|
||
if (this.debugConfig.debugUi) {
|
||
this.debugConfig?.debugUi.destroyContainer();
|
||
this.debugConfig.debugUi = undefined;
|
||
}
|
||
}
|
||
//#endregion
|
||
|
||
/**
|
||
* Checks whether autodetection can run
|
||
*/
|
||
startCheck(arVariant?: ArVariant) {
|
||
this.arVariant = arVariant;
|
||
|
||
if (!this.videoData.player) {
|
||
// console.warn('Player not detected!');
|
||
// console.log('--- video data: ---\n', this.videoData);
|
||
return;
|
||
}
|
||
if (this.siteSettings.canRunAard(this.videoData.player.environment)) {
|
||
this.start();
|
||
} else {
|
||
this.stop();
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Clears autoDisable flag
|
||
*/
|
||
clearAutoDisabled() {
|
||
this.status.autoDisabled = false;
|
||
this.timers.autoDisableAt = undefined;
|
||
}
|
||
|
||
/**
|
||
* Starts autodetection loop.
|
||
*/
|
||
start() {
|
||
this.clearAutoDisabled();
|
||
this.forceFullRecheck = true;
|
||
if (this.videoData.resizer.lastAr.type === AspectRatioType.AutomaticUpdate) {
|
||
// ensure first autodetection will run in any case
|
||
this.videoData.resizer.lastAr = {type: AspectRatioType.AutomaticUpdate, ratio: this.defaultAr};
|
||
}
|
||
|
||
// do full reset of test samples
|
||
this.testResults = initAardTestResults(this.settings.active.aard);
|
||
this.verticalTestResults = initAardTestResults(this.settings.active.aard);
|
||
|
||
if (this.animationFrame) {
|
||
window.cancelAnimationFrame(this.animationFrame);
|
||
}
|
||
|
||
this.status.aardActive = true;
|
||
this.animationFrame = window.requestAnimationFrame( (ts: DOMHighResTimeStamp) => this.onAnimationFrame(ts));
|
||
|
||
// set auto-disable timer if detection timeout is set
|
||
if (this.settings.active.aard.autoDisable.ifNotChanged) {
|
||
this.timers.autoDisableAt = Date.now() + this.settings.active.aard.autoDisable.ifNotChangedTimeout;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Runs autodetection ONCE.
|
||
* If autodetection loop is running, this will also stop autodetection loop.
|
||
*/
|
||
step(options?: {noCache?: boolean}) {
|
||
this.stop();
|
||
|
||
if (options?.noCache) {
|
||
this.testResults = initAardTestResults(this.settings.active.aard);
|
||
this.verticalTestResults = initAardTestResults(this.settings.active.aard);
|
||
}
|
||
|
||
this.main();
|
||
}
|
||
|
||
/**
|
||
* Stops autodetection.
|
||
*/
|
||
stop() {
|
||
if (this.animationFrame) {
|
||
window.cancelAnimationFrame(this.animationFrame);
|
||
}
|
||
}
|
||
|
||
//#region animationFrame, scheduling, and other shit
|
||
/**
|
||
* Checks whether conditions for granting a frame check are fulfilled
|
||
* @returns
|
||
*/
|
||
private canTriggerFrameCheck() {
|
||
|
||
// console.log('ard check status:', this.status);
|
||
|
||
// if video was paused & we know that we already checked that frame,
|
||
// we will not check it again.
|
||
const videoState = this.getVideoPlaybackState();
|
||
const polling = this.settings.active.aard.polling;
|
||
const now = Date.now();
|
||
|
||
if (videoState !== VideoPlaybackState.Playing) {
|
||
if (this.status.lastVideoStatus === videoState) {
|
||
return false;
|
||
}
|
||
}
|
||
if (this.status.autoDisabled) {
|
||
return false;
|
||
}
|
||
if (this.timers.autoDisableAt < now) {
|
||
this.status.autoDisabled = true;
|
||
return false;
|
||
}
|
||
this.status.lastVideoStatus = videoState;
|
||
|
||
const tabVisible = document.visibilityState === 'visible';
|
||
if (!tabVisible && polling.runInBackgroundTabs === AardPollingOptions.No) {
|
||
return false;
|
||
}
|
||
if (this.videoData.player.isTooSmall && polling.runOnSmallVideos === AardPollingOptions.No) {
|
||
return false;
|
||
}
|
||
|
||
const isActive = (tabVisible || polling.runInBackgroundTabs !== AardPollingOptions.Reduced)
|
||
&& (!this.videoData.player.isTooSmall || polling.runOnSmallVideos !== AardPollingOptions.Reduced);
|
||
const nextCheck = isActive ? this.timers.nextFrameCheckTime : this.timers.reducedPollingNextCheckTime;
|
||
|
||
if (now < nextCheck) {
|
||
return false;
|
||
}
|
||
|
||
this.timers.nextFrameCheckTime = now + this.settings.active.aard.timers.playing;
|
||
this.timers.reducedPollingNextCheckTime = now + this.settings.active.aard.timers.playingReduced;
|
||
return true;
|
||
}
|
||
|
||
/**
|
||
* Bootstraps the main loop.
|
||
*
|
||
* Honestly this doesn't need description, but I want to put some green
|
||
* between two adjacent functions.
|
||
*/
|
||
private onAnimationFrame(ts: DOMHighResTimeStamp) {
|
||
if (this.canTriggerFrameCheck()) {
|
||
resetAardTestResults(this.testResults);
|
||
resetSamples(this.canvasSamples);
|
||
resetSubtitleScanResults(this.testResults);
|
||
this.main();
|
||
this.forceFullRecheck = false;
|
||
} else {
|
||
}
|
||
this.animationFrame = window.requestAnimationFrame( (ts: DOMHighResTimeStamp) => this.onAnimationFrame(ts));
|
||
}
|
||
//#endregion
|
||
|
||
/**
|
||
* Main loop for scanning aspect ratio changes
|
||
*/
|
||
private async main() {
|
||
const arConf = this.settings.active.aard;
|
||
|
||
try {
|
||
this.timer.next();
|
||
|
||
let imageData: Uint8Array;
|
||
this.timer.current.start = performance.now();
|
||
|
||
// We abuse a do-while loop to eat our cake (get early returns)
|
||
// and have it, too (if we return early, we still execute code
|
||
// at the end of this function)
|
||
scanFrame:
|
||
{
|
||
imageData = await new Promise<Uint8Array>(
|
||
resolve => {
|
||
try {
|
||
this.canvasStore.main.drawVideoFrame(this.video);
|
||
this.timer.current.draw = performance.now() - this.timer.current.start;
|
||
resolve(this.canvasStore.main.getImageData());
|
||
} catch (e) {
|
||
if (e.name === 'SecurityError') {
|
||
this.eventBus.send('uw-config-broadcast', {type: 'aard-error', aardErrors: {cors: true}});
|
||
this.stop();
|
||
}
|
||
if (this.canvasStore.main instanceof FallbackCanvas) {
|
||
if (this.inFallback) {
|
||
this.eventBus.send('uw-config-broadcast', {type: 'aard-error', aardErrors: this.fallbackReason});
|
||
this.stop();
|
||
} else {
|
||
this.eventBus.send('uw-config-broadcast', {type: 'aard-error', aardErrors: {fallbackCanvasError: true}});
|
||
this.stop();
|
||
}
|
||
} else {
|
||
if (arConf.aardType === 'auto') {
|
||
this.canvasStore.main.destroy();
|
||
this.canvasStore.main = this.createCanvas('main-gl', 'legacy');
|
||
}
|
||
this.inFallback = true;
|
||
this.fallbackReason = {cors: true};
|
||
|
||
if (arConf.aardType !== 'auto') {
|
||
this.stop();
|
||
}
|
||
}
|
||
}
|
||
}
|
||
);
|
||
this.timer.current.getImage = performance.now() - this.timer.current.start;
|
||
|
||
// STEP 1:
|
||
// Test if corners are black. If they're not, we can immediately quit the loop.
|
||
// For performances of measurements, checking orientation of letterbox is part of fastBlackLevel
|
||
const lastValidLetterboxOrientation = this.testResults.lastValidLetterboxOrientation;
|
||
|
||
orientationCheck:
|
||
{
|
||
this.getBlackLevelFast(
|
||
imageData, 3, 1,
|
||
arConf.canvasDimensions.sampleCanvas.width,
|
||
arConf.canvasDimensions.sampleCanvas.height
|
||
);
|
||
|
||
if (this.testResults.letterboxOrientation === LetterboxOrientation.NotLetterbox) {
|
||
break orientationCheck;
|
||
}
|
||
|
||
this.letterboxOrientationScan(
|
||
imageData,
|
||
arConf.canvasDimensions.sampleCanvas.width,
|
||
arConf.canvasDimensions.sampleCanvas.height
|
||
);
|
||
}
|
||
|
||
|
||
this.timer.current.fastBlackLevel = performance.now() - this.timer.current.start;
|
||
|
||
// if we detect no letterbox, we don't test anything — instead, we immediately reset
|
||
if (this.testResults.letterboxOrientation === LetterboxOrientation.NotLetterbox) {
|
||
// TODO: reset aspect ratio to "AR not applied"
|
||
this.testResults.lastStage = 1;
|
||
this.testResults.letterboxSize = 0;
|
||
this.testResults.letterboxOffset = 0;
|
||
resetGuardLine(this.testResults);
|
||
|
||
break scanFrame;
|
||
}
|
||
|
||
// If we detect both letterbox and pillarbox, we keep things as they are but avoid scanning further
|
||
if (this.testResults.letterboxOrientation === LetterboxOrientation.Both) {
|
||
this.testResults.lastStage = 1;
|
||
|
||
break scanFrame;
|
||
}
|
||
|
||
// If lastValidLetterboxOrientation changed, we reset guard line (& gang), but continue processing
|
||
if (lastValidLetterboxOrientation !== this.testResults.lastValidLetterboxOrientation) {
|
||
this.forceFullRecheck = true;
|
||
}
|
||
|
||
// We only do subtitle check when orientation is letterbox
|
||
if (this.testResults.letterboxOrientation === LetterboxOrientation.Letterbox) {
|
||
this.subtitleScan(
|
||
imageData,
|
||
arConf.canvasDimensions.sampleCanvas.width,
|
||
arConf.canvasDimensions.sampleCanvas.height,
|
||
false,
|
||
);
|
||
}
|
||
|
||
}
|
||
|
||
// Note that subtitle check should reset aspect ratio outright, regardless of what other tests revealed.
|
||
// Also note that subtitle check should run on newest aspect ratio data, rather than lag one frame behind
|
||
// But implementation details are something for future Tam to figure out
|
||
|
||
// If forceFullRecheck is set, then 'not letterbox' should always force-reset the aspect ratio
|
||
// (as aspect ratio may have been set manually while autodetection was off)
|
||
|
||
// If debugging is enable,
|
||
this.canvasStore.debug?.drawBuffer(imageData);
|
||
|
||
processUpdate:
|
||
{
|
||
if (this.testResults.letterboxOrientation === LetterboxOrientation.NotLetterbox) {
|
||
// console.warn('DETECTED NOT LETTERBOX! (resetting)')
|
||
this.timer.arChanged();
|
||
this.updateAspectRatio(this.defaultAr, {forceReset: true});
|
||
this.testResults.activeLetterbox.width = 0;
|
||
this.testResults.activeLetterbox.offset = 0;
|
||
this.testResults.activeLetterbox.orientation = LetterboxOrientation.NotLetterbox;
|
||
break processUpdate;
|
||
}
|
||
|
||
if (this.testResults.subtitleDetected && arConf.subtitles.subtitleCropMode !== AardSubtitleCropMode.CropSubtitles) {
|
||
if (arConf.subtitles.subtitleCropMode === AardSubtitleCropMode.ResetAR) {
|
||
this.updateAspectRatio(this.defaultAr, {forceReset: true});
|
||
this.testResults.activeLetterbox.width = 0;
|
||
this.testResults.activeLetterbox.offset = 0;
|
||
this.testResults.activeLetterbox.orientation = LetterboxOrientation.NotLetterbox;
|
||
this.timers.pauseUntil = Date.now() + arConf.subtitles.resumeAfter;
|
||
|
||
} else if (arConf.subtitles.subtitleCropMode === AardSubtitleCropMode.ResetAndDisable) {
|
||
this.updateAspectRatio(this.defaultAr, {forceReset: true});
|
||
this.testResults.activeLetterbox.width = 0;
|
||
this.testResults.activeLetterbox.offset = 0;
|
||
this.testResults.activeLetterbox.orientation = LetterboxOrientation.NotLetterbox;
|
||
|
||
this.status.autoDisabled = true;
|
||
}
|
||
|
||
break processUpdate;
|
||
}
|
||
|
||
// if detection is uncertain, we don't do anything at all (unless if guardline was broken, in which case we reset)
|
||
if (this.testResults.aspectRatioUncertain) {
|
||
// this.timer.arChanged();
|
||
|
||
// if (
|
||
// this.testResults.aspectRatioCheck.frontCandidate < this.testResults.guardLine.front
|
||
// || this.testResults.aspectRatioCheck.backCandidate > this.testResults.guardLine.back
|
||
// || this.testResults.aspectRatioCheck.frontCandidate === -1
|
||
// || this.testResults.aspectRatioCheck.backCandidate === -1
|
||
// ) {
|
||
// this.updateAspectRatio(this.defaultAr, {uncertainDetection: true, forceReset: true});
|
||
// }
|
||
|
||
break processUpdate;
|
||
}
|
||
|
||
// TODO: emit debug values if debugging is enabled
|
||
this.testResults.isFinished = true;
|
||
|
||
this.testResults.guardLine.front = this.testResults.aspectRatioCheck.frontCandidate;
|
||
this.testResults.guardLine.back = this.testResults.aspectRatioCheck.backCandidate;
|
||
|
||
// TODO: set flag if subtitles are far enough from edge to avoid getting cropped
|
||
const finalAr = this.getAr();
|
||
if (finalAr > 0) {
|
||
this.updateAspectRatio(finalAr);
|
||
this.testResults.activeLetterbox.width = this.testResults.letterboxSize;
|
||
this.testResults.activeLetterbox.offset = this.testResults.letterboxOffset;
|
||
this.testResults.activeLetterbox.orientation = this.testResults.letterboxOrientation;
|
||
} else {
|
||
this.testResults.aspectRatioInvalid = true;
|
||
this.testResults.aspectRatioInvalidReason = finalAr.toFixed(3);
|
||
}
|
||
// }
|
||
|
||
// if we got "no letterbox" OR aspectRatioUpdated
|
||
}
|
||
|
||
if (this.canvasStore.debug) {
|
||
// this.canvasStore.debug.drawBuffer(imageData);
|
||
this.timer.getAverage();
|
||
this.debugConfig?.debugUi?.updateTestResults(this.testResults, this.timers);
|
||
}
|
||
} catch (e) {
|
||
console.warn('[Ultrawidify] Aspect ratio autodetection crashed for some reason.\n\nsome reason:', e);
|
||
this.videoData.resizer.setAr({type: AspectRatioType.AutomaticUpdate, ratio: this.defaultAr, variant: this.arVariant});
|
||
}
|
||
}
|
||
|
||
private getVideoPlaybackState(): VideoPlaybackState {
|
||
try {
|
||
if (this.video.ended) {
|
||
return VideoPlaybackState.Ended;
|
||
} else if (this.video.paused) {
|
||
return VideoPlaybackState.Paused;
|
||
} else if (this.video.error) {
|
||
return VideoPlaybackState.Error;
|
||
} else {
|
||
return VideoPlaybackState.Playing;
|
||
}
|
||
} catch (e) {
|
||
this.logger.warn('getVideoPlaybackState]', `There was an error while determining video playback state.`, e);
|
||
return VideoPlaybackState.Error;
|
||
}
|
||
}
|
||
|
||
|
||
//#region buffer tests
|
||
/**
|
||
* Get black level of a given frame. We sample black level on very few
|
||
* positions — just the corners of the frame. If letterboxing or pillarboxing
|
||
* exists, then pixels in the corners of the frame should be the blackest
|
||
* it gets.
|
||
*
|
||
* Sampling pattern are four lines, each shooting from its respective corner.
|
||
* Value of 'sample' parameter determines how many pixels along this line we
|
||
* are going to sample. Offset means how many pixels of those four lines we
|
||
* are going to skip before we start sampling.
|
||
*
|
||
* x→ 0 1 ... ... x-1
|
||
* y↓ × ------------... ...------------ ×
|
||
* 0 | 1 1 |
|
||
* 1 | 2 2 |
|
||
* : | . . :
|
||
* : . .
|
||
*
|
||
* : . . :
|
||
* | . . |
|
||
* | 2 2 |
|
||
* h-1 | 1 1 |
|
||
* × ------------... ...------------ ×
|
||
*
|
||
*
|
||
* IMPORTANT NOTES
|
||
* <> imageData is one-dimensional array, so we need to account for that.
|
||
* <> blackLevel is the darkest brightest subpixel detected
|
||
* <> If image has no crop, then this function WILL NOT get the true black level.
|
||
* In that case, we don't get an accurate black level, but we know straight
|
||
* away that the image is uncropped. If image is uncropped, we can skip other,
|
||
* more expensive tests.
|
||
*
|
||
* @param imageData array of pixels (4 bytes/fields per pixel)
|
||
* @param samples number of samples per corner
|
||
* @param width width of the frame
|
||
* @param height height of the frame
|
||
*/
|
||
private getBlackLevelFast(imageData: Uint8Array, samples: number, offset: number, width: number, height: number) {
|
||
// there's 4 points for each sample, and 3 components for each of the sampling points.
|
||
const pixelValues = new Array<number>(samples * 12);
|
||
let pvi = 0;
|
||
|
||
/**
|
||
* We should ensure we are accessing pixels in ordered manner in order to
|
||
* take advantage of data locality.
|
||
*/
|
||
const end = offset + samples;
|
||
for (let i = offset; i < end; i++) {
|
||
const px_r = (i * ROW_SIZE) + (i * PIXEL_SIZE); // red component starts here
|
||
pixelValues[pvi++] = imageData[px_r];
|
||
pixelValues[pvi++] = imageData[px_r + 1];
|
||
pixelValues[pvi++] = imageData[px_r + 2];
|
||
imageData[px_r + 3] = GlDebugType.BlackLevelSample;
|
||
|
||
const endpx_r = px_r + ROW_SIZE - (i * PIXEL_SIZE * 2) - PIXEL_SIZE; // - twice the offset to mirror the diagonal
|
||
pixelValues[pvi++] = imageData[endpx_r];
|
||
pixelValues[pvi++] = imageData[endpx_r + 1];
|
||
pixelValues[pvi++] = imageData[endpx_r + 2];
|
||
imageData[endpx_r + 3] = GlDebugType.BlackLevelSample;
|
||
}
|
||
|
||
// now let's populate the bottom two corners
|
||
for (let i = end; i --> offset;) {
|
||
const row = height - i - 1; // since first row is 0, last row is height - 1
|
||
|
||
const px_r = (row * ROW_SIZE) + (i * PIXEL_SIZE);
|
||
pixelValues[pvi++] = imageData[px_r];
|
||
pixelValues[pvi++] = imageData[px_r + 1];
|
||
pixelValues[pvi++] = imageData[px_r + 2];
|
||
imageData[px_r + 3] = GlDebugType.BlackLevelSample;
|
||
|
||
const endpx_r = px_r + (ROW_SIZE) - (i * PIXEL_SIZE * 2) - PIXEL_SIZE; // - twice the offset to mirror the diagonal
|
||
pixelValues[pvi++] = imageData[endpx_r];
|
||
pixelValues[pvi++] = imageData[endpx_r + 1];
|
||
pixelValues[pvi++] = imageData[endpx_r + 2];
|
||
imageData[endpx_r + 3] = GlDebugType.BlackLevelSample;
|
||
}
|
||
|
||
let min = 255;
|
||
let avg = 0;
|
||
let p = 0;
|
||
|
||
for (let i = 0; i < pixelValues.length; i++) {
|
||
p = pixelValues[i];
|
||
i++;
|
||
|
||
if (p < pixelValues[i]) {
|
||
p = pixelValues[i];
|
||
}
|
||
i++;
|
||
|
||
if (p < pixelValues[i]) {
|
||
p = pixelValues[i];
|
||
}
|
||
|
||
avg += p;
|
||
if (p < min) {
|
||
min = p;
|
||
}
|
||
}
|
||
|
||
// While there's 4 bytes / 3 values per pixel, a
|
||
// avg only contains highest subpixel ... so we really
|
||
// only take one sample per pixel instead of 3/4
|
||
avg = avg / samples;
|
||
|
||
if (avg > this.testResults.blackThreshold) {
|
||
this.testResults.letterboxOrientation = LetterboxOrientation.NotLetterbox;
|
||
}
|
||
|
||
// only update black level if not letterbox.
|
||
// NOTE: but maybe we could, if blackLevel can only get lower than
|
||
// the default value.
|
||
if (this.testResults.letterboxOrientation === LetterboxOrientation.NotLetterbox) {
|
||
this.testResults.aspectRatioUncertain = false;
|
||
}
|
||
|
||
if (min < this.testResults.blackLevel) {
|
||
this.testResults.blackLevel = min;
|
||
this.testResults.blackThreshold = min + 16;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Checks orientation of black bars.
|
||
* @param imageData
|
||
* @param width
|
||
* @param height
|
||
*/
|
||
private letterboxOrientationScan(imageData: Uint8Array, width: number, height: number) {
|
||
const lastPixelOffset = ROW_SIZE - PIXEL_SIZE;
|
||
const imageSize = ROW_SIZE * height;
|
||
|
||
const xLimit = this.settings.active.aard.letterboxOrientationScan.letterboxLimit;
|
||
const yLimit = this.settings.active.aard.letterboxOrientationScan.pillarboxLimit;
|
||
|
||
let letterbox = true, pillarbox = true;
|
||
let xCount = 0, yCount = 0;
|
||
|
||
// scan top row
|
||
for (let i = 0; i < ROW_SIZE; i += PIXEL_SIZE) {
|
||
if (
|
||
imageData[i ] > this.testResults.blackThreshold
|
||
|| imageData[i+1] > this.testResults.blackThreshold
|
||
|| imageData[i+2] > this.testResults.blackThreshold
|
||
) {
|
||
imageData[i + 3] = GlDebugType.LetterboxOrientationScanImageDetection
|
||
if (++xCount > xLimit) {
|
||
letterbox = false;
|
||
break;
|
||
}
|
||
} else {
|
||
imageData[i+3] = GlDebugType.LetterboxOrientationScanTrace
|
||
}
|
||
}
|
||
|
||
// scan sides
|
||
for (let i = 0; i < imageSize; i += ROW_SIZE) {
|
||
const lastPx = i + lastPixelOffset;
|
||
|
||
// left side
|
||
if (
|
||
imageData[i ] > this.testResults.blackThreshold
|
||
|| imageData[i+1] > this.testResults.blackThreshold
|
||
|| imageData[i+2] > this.testResults.blackThreshold
|
||
) {
|
||
imageData[i+3] = GlDebugType.LetterboxOrientationScanImageDetection;
|
||
if (++yCount > yLimit) {
|
||
pillarbox = false;
|
||
break;
|
||
}
|
||
} else {
|
||
imageData[i+3] = GlDebugType.LetterboxOrientationScanTrace;
|
||
}
|
||
|
||
// right side
|
||
if (
|
||
imageData[lastPx ] > this.testResults.blackThreshold
|
||
|| imageData[lastPx+1] > this.testResults.blackThreshold
|
||
|| imageData[lastPx+2] > this.testResults.blackThreshold
|
||
) {
|
||
imageData[lastPx+3] = GlDebugType.LetterboxOrientationScanImageDetection;
|
||
if (++yCount > yLimit) {
|
||
pillarbox = false;
|
||
break;
|
||
}
|
||
} else {
|
||
imageData[lastPx+3] = GlDebugType.LetterboxOrientationScanTrace;
|
||
}
|
||
}
|
||
|
||
// scan bottom row
|
||
if (letterbox) {
|
||
for (let i = ROW_SIZE * (height - 1); i < imageSize; i += PIXEL_SIZE) {
|
||
if ( imageData[i ] > this.testResults.blackThreshold
|
||
|| imageData[i+1] > this.testResults.blackThreshold
|
||
|| imageData[i+2] > this.testResults.blackThreshold
|
||
) {
|
||
imageData[i + 3] = GlDebugType.LetterboxOrientationScanImageDetection
|
||
if (++xCount > xLimit) {
|
||
letterbox = false;
|
||
break;
|
||
}
|
||
} else {
|
||
imageData[i+3] = GlDebugType.LetterboxOrientationScanTrace
|
||
}
|
||
}
|
||
}
|
||
|
||
// determine result
|
||
if (letterbox && pillarbox) {
|
||
this.testResults.letterboxOrientation = LetterboxOrientation.Both;
|
||
} else if (letterbox) {
|
||
this.testResults.letterboxOrientation = LetterboxOrientation.Letterbox;
|
||
this.testResults.lastValidLetterboxOrientation = LetterboxOrientation.Letterbox;
|
||
} else if (pillarbox) {
|
||
this.testResults.letterboxOrientation = LetterboxOrientation.Pillarbox;
|
||
this.testResults.lastValidLetterboxOrientation = LetterboxOrientation.Pillarbox;
|
||
} else {
|
||
this.testResults.letterboxOrientation = LetterboxOrientation.NotLetterbox;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Updates letterbox edge (updates imageLine and guardLine)
|
||
* @param crossDimension height of the sample frame (or width, if pillarbox)
|
||
* @param topCandidate First line of image data on the top of the frame
|
||
* @param bottomCandidate First line with image data on the bottom of the frame
|
||
* @returns
|
||
*/
|
||
private updateLetterboxEdgeCandidates(crossDimension: number, topCandidate: number, bottomCandidate: number) {
|
||
// if new topCandidate or bottomCandidate aren't valid,
|
||
// use existing value.
|
||
if (topCandidate < 0) {
|
||
topCandidate = this.testResults.aspectRatioCheck.frontCandidate;
|
||
}
|
||
if (bottomCandidate < 0) {
|
||
bottomCandidate = this.testResults.aspectRatioCheck.backCandidate;
|
||
}
|
||
|
||
const bottomDistance = (crossDimension - bottomCandidate);
|
||
const maxOffset = ~~(crossDimension * this.settings.active.aard.edgeDetection.maxLetterboxOffset);
|
||
const diff = Math.abs(topCandidate - bottomDistance);
|
||
const candidateAvg = ~~((topCandidate + bottomDistance) * 0.5);
|
||
|
||
this.testResults.aspectRatioCheck.frontCandidate = topCandidate;
|
||
this.testResults.aspectRatioCheck.backCandidate = bottomCandidate;
|
||
|
||
if (diff > maxOffset) {
|
||
this.testResults.aspectRatioUncertain = true;
|
||
this.testResults.aspectRatioUncertainReason = AardUncertainReason.LetterboxNotCenteredEnough;
|
||
}
|
||
|
||
this.testResults.letterboxSize = candidateAvg;
|
||
this.testResults.letterboxOffset = diff;
|
||
|
||
if (this.testResults.letterboxOrientation === LetterboxOrientation.Letterbox && this.testResults.subtitleDetected) {
|
||
const top = this.testResults.subtitleScan.regions.top.firstSubtitle === -1 ? topCandidate : Math.min(topCandidate, this.testResults.subtitleScan.regions.top.firstSubtitle);
|
||
const bottom = Math.max(bottomCandidate, this.testResults.subtitleScan.regions.bottom.firstSubtitle);
|
||
|
||
this.testResults.letterboxSizeWithSubtitles = ~~((top + (crossDimension - bottomCandidate)) * 0.5);
|
||
}
|
||
}
|
||
|
||
|
||
/**
|
||
* Scans for subtitles
|
||
* @param imageData
|
||
* @param width
|
||
* @param height
|
||
* @returns
|
||
*/
|
||
private subtitleScan(imageData: Uint8Array, width: number, height: number, skipAdvancedScan: boolean) {
|
||
const scanConf = this.settings.active.aard.subtitles;
|
||
|
||
this.testResults.subtitleDetected = false;
|
||
|
||
// SubtitleScanResult Regions
|
||
const ssrRegions = this.testResults.subtitleScan.regions;
|
||
const halfHeight = Math.floor(height / 2);
|
||
|
||
resetSubtitleScanResults(this.testResults);
|
||
|
||
this.subtitleScanRegionIterative(
|
||
imageData, height,
|
||
2, halfHeight,
|
||
scanConf.refiningScanSpacing, scanConf.minDetections,
|
||
ssrRegions.top,
|
||
)
|
||
|
||
this.subtitleScanRegionIterative(
|
||
imageData, height,
|
||
height - 3, halfHeight,
|
||
-scanConf.refiningScanSpacing, scanConf.minDetections,
|
||
ssrRegions.bottom,
|
||
)
|
||
|
||
|
||
if (ssrRegions.top.uncertain || ssrRegions.bottom.uncertain) {
|
||
this.testResults.aspectRatioUncertain = true;
|
||
} else {
|
||
this.testResults.aspectRatioUncertain = false;
|
||
}
|
||
|
||
// 1. updateLetterboxEdgeCandidates runs regardless of whether we detected subtitles or not.
|
||
// 2. it's also not affected by whether subtitleDetected is set
|
||
// 3. we actually need to only reset letterbox if subtitle is actually outside of the crop area
|
||
this.updateLetterboxEdgeCandidates(
|
||
height,
|
||
ssrRegions.top.firstImage,
|
||
ssrRegions.bottom.firstImage
|
||
);
|
||
|
||
|
||
// we can only do this in actual letterbox
|
||
if (this.testResults.activeLetterbox.orientation === LetterboxOrientation.Letterbox) {
|
||
|
||
// we only reset letterbox if letters are outside the video area, otherwise we risk
|
||
// getting whacked by credits, ppt youtubers, and other shit like that
|
||
const borderTop = this.testResults.activeLetterbox.width;
|
||
const borderBottom = this.settings.active.aard.canvasDimensions.sampleCanvas.height - this.testResults.activeLetterbox.width;
|
||
|
||
if (
|
||
(ssrRegions.top.firstSubtitle !== -1 && ssrRegions.top.firstSubtitle < borderTop)
|
||
|| (ssrRegions.bottom.firstSubtitle !== -1 && ssrRegions.bottom.firstSubtitle > borderBottom)
|
||
) {
|
||
|
||
this.testResults.subtitleDetected = true;
|
||
}
|
||
|
||
}
|
||
|
||
|
||
this.timer.current.subtitleScan = performance.now() - this.timer.current.start;
|
||
}
|
||
|
||
/**
|
||
* Scans region of video frame for presence of subtitles
|
||
* @param imageData
|
||
* @param startRow
|
||
* @param endRow
|
||
* @param minDetections
|
||
* @param results
|
||
*/
|
||
private subtitleScanRegionLinear(
|
||
imageData: Uint8Array,
|
||
height: number,
|
||
startRow: number,
|
||
endRow: number,
|
||
scanSpacing: number,
|
||
minDetections: number,
|
||
results: AardTestResult_SubtitleRegion,
|
||
) {
|
||
results.uncertain = false;
|
||
|
||
const scanConf = this.settings.active.aard.subtitles;
|
||
const arConf = this.settings.active.aard;
|
||
|
||
let letterCount, imageSegmentCount, potentialFadedLetterCount, potentialFadedLetterCountInvalidated, nonGradientPixelCount, letterSize, imageSize, imageSegmentSize, imageWeightedSize, segmentWeights, imageSegmentAlignment, imageSegmentAlignmentSamples,
|
||
isOnLetter, isOnImage, isBlank,
|
||
gradientRowOffset_before, gradientRowOffset_after;
|
||
let rowStart, rowEnd, rowMid, rowGTA, rowGTB; // GT = gradient test
|
||
let imageConfirmPass = false, subtitleConfirmPass = false;
|
||
|
||
let outerIteration = 0;
|
||
|
||
const rowMargin = Math.floor(scanConf.scanMargin * ROW_SIZE);
|
||
const imageThreshold = Math.floor((ROW_SIZE - (rowMargin * 2)) * arConf.edgeDetection.minValidImage * PIXEL_SIZE_FRACTION);
|
||
|
||
|
||
// search in top letterbox
|
||
outerLoop:
|
||
for (
|
||
let searchRow = startRow;
|
||
(scanSpacing > 0 && searchRow < endRow) || (scanSpacing < 0 && searchRow > endRow);
|
||
searchRow += scanSpacing
|
||
) {
|
||
if (++outerIteration > height) {
|
||
// console.warn('[ultrawidify|aard::subtitleScanRegionLinear] — scan got stuck in an infinite loop. This shouldn\'t happen.');
|
||
results.uncertain;
|
||
break outerLoop;
|
||
}
|
||
|
||
letterCount = 0;
|
||
potentialFadedLetterCount = 0;
|
||
imageSegmentCount = 0;
|
||
imageSegmentSize = 0;
|
||
nonGradientPixelCount = 0;
|
||
|
||
imageSize = 0;
|
||
imageWeightedSize = 0;
|
||
segmentWeights = 0;
|
||
|
||
letterSize = 0;
|
||
isOnLetter = false;
|
||
isOnImage = false;
|
||
isBlank = true;
|
||
potentialFadedLetterCountInvalidated = false;
|
||
|
||
imageSegmentAlignment = 0;
|
||
imageSegmentAlignmentSamples = 0;
|
||
|
||
// Scan region is centered,
|
||
rowStart = (searchRow * ROW_SIZE) + rowMargin;
|
||
|
||
if (scanSpacing > 0) {
|
||
gradientRowOffset_before = (Math.max(searchRow - 2, 0) * ROW_SIZE);
|
||
gradientRowOffset_after = (Math.min(searchRow + 2, height) * ROW_SIZE);
|
||
} else {
|
||
gradientRowOffset_before = (Math.max(searchRow - 2, 0) * ROW_SIZE);
|
||
gradientRowOffset_after = (Math.min(searchRow + 2, height) * ROW_SIZE);
|
||
}
|
||
|
||
// exact row doesn't matter ... unless scanMargin is 0
|
||
rowEnd = ((searchRow + 1) * ROW_SIZE) - rowMargin;
|
||
rowMid = (rowStart + rowEnd) * 0.5;
|
||
|
||
const resetSubtitlePass =() => {
|
||
if (subtitleConfirmPass) {
|
||
subtitleConfirmPass = false;
|
||
searchRow -= (scanSpacing > 0 ? + 1 : -1);
|
||
}
|
||
}
|
||
|
||
const updateImage = (candidate: number) => {
|
||
if (scanSpacing > 0) {
|
||
if (results.firstImage === -1 || candidate < results.firstImage) {
|
||
results.firstImage = candidate;
|
||
}
|
||
} else {
|
||
if (results.firstImage === -1 || candidate > results.firstImage) {
|
||
results.firstImage = candidate;
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* This function can break or return early only in the following situations:
|
||
*
|
||
* * we have enough letters to detect subtitles — continue outerLoop
|
||
* * Single "letter" is too wide — immediate return, as we found where the image starts
|
||
*
|
||
* Other instances require a bit more complex analysis.
|
||
*/
|
||
|
||
while (rowStart < rowEnd) {
|
||
const r = imageData[rowStart], g = imageData[rowStart + 1], b = imageData[rowStart + 2];
|
||
|
||
const on = r > scanConf.subtitleSubpixelThresholdOn
|
||
|| g > scanConf.subtitleSubpixelThresholdOn
|
||
|| b > scanConf.subtitleSubpixelThresholdOn;
|
||
const off = r < scanConf.subtitleSubpixelThresholdOff
|
||
&& g < scanConf.subtitleSubpixelThresholdOff
|
||
&& b < scanConf.subtitleSubpixelThresholdOff;
|
||
|
||
if (off) {
|
||
imageData[rowStart + 3] = GlDebugType.SubtitleThresholdOff;
|
||
|
||
|
||
// if isOnLetter was set, that means we've just concluded a letter segment
|
||
if (isOnLetter) {
|
||
letterCount++;
|
||
|
||
if (letterCount > minDetections) {
|
||
if (results.firstSubtitle === -1) {
|
||
results.firstSubtitle = searchRow;
|
||
|
||
// if detecting subtitles only resets AR, we can return immediately
|
||
if (scanConf.subtitleCropMode !== AardSubtitleCropMode.CropSubtitles) {
|
||
break outerLoop;
|
||
}
|
||
}
|
||
results.lastSubtitle = searchRow;
|
||
isBlank = false;
|
||
|
||
imageConfirmPass = false;
|
||
continue outerLoop;
|
||
}
|
||
}
|
||
if (isOnImage) {
|
||
if (imageSegmentSize < scanConf.maxValidLetter) {
|
||
potentialFadedLetterCount++;
|
||
|
||
// track potential letter alignment
|
||
imageSegmentAlignment += (rowStart - rowMid) * imageSegmentSize * PIXEL_SIZE_FRACTION;
|
||
imageSegmentAlignmentSamples += imageSegmentSize;
|
||
} else {
|
||
potentialFadedLetterCountInvalidated = true;
|
||
}
|
||
|
||
if (imageSegmentSize > arConf.edgeDetection.minEdgeSegmentSize) {
|
||
imageSegmentCount++;
|
||
imageWeightedSize += imageSegmentSize * imageSegmentSize; // longer segments should have bigger weight
|
||
segmentWeights += imageSegmentSize;
|
||
imageSegmentSize = 0;
|
||
}
|
||
}
|
||
|
||
isOnLetter = false;
|
||
isOnImage = false;
|
||
letterSize = 0;
|
||
imageSegmentSize = 0;
|
||
} else {
|
||
imageData[rowStart + 3] = GlDebugType.SubtitleThresholdNone;
|
||
isOnImage = true;
|
||
isBlank = false;
|
||
imageSegmentSize++;
|
||
imageSize++;
|
||
|
||
// let's see if we're accidentally detecting gradient.
|
||
// We only need to run these tests when pixels are dark.
|
||
// We keep track of number of pixels that pass the test, or don't
|
||
// need the test to begin with.
|
||
if (!on && (
|
||
imageData[rowStart ] < arConf.edgeDetection.gradientThreshold
|
||
&& imageData[rowStart + 1] < arConf.edgeDetection.gradientThreshold
|
||
&& imageData[rowStart + 2] < arConf.edgeDetection.gradientThreshold
|
||
)) {
|
||
rowGTB = rowStart - gradientRowOffset_before;
|
||
rowGTA = rowStart + gradientRowOffset_after;
|
||
|
||
// if true, then gradient.
|
||
// The first row gives technically incorrect answers for pixels directly under subtitles, but since
|
||
// "nonGradientPixelCount" is a shorthand for "we're relatively confident in this detection", potentially
|
||
// mistaking non-black pixels under subtitles for gradient isn't too problematic
|
||
if ( (imageData[rowStart ] - imageData[rowGTB ]) < arConf.edgeDetection.gradientTestMinDelta
|
||
&& (imageData[rowStart + 1] - imageData[rowGTB + 1]) < arConf.edgeDetection.gradientTestMinDelta
|
||
&& (imageData[rowStart + 2] - imageData[rowGTB + 2]) < arConf.edgeDetection.gradientTestMinDelta
|
||
&& imageData[rowGTA ] - imageData[rowStart ] > arConf.edgeDetection.gradientTestMinDeltaAfter
|
||
&& imageData[rowGTA + 1] - imageData[rowStart + 1] > arConf.edgeDetection.gradientTestMinDeltaAfter
|
||
&& imageData[rowGTA + 2] - imageData[rowStart + 2] > arConf.edgeDetection.gradientTestMinDeltaAfter
|
||
&& imageData[rowGTA ] - imageData[rowStart ] < arConf.edgeDetection.gradientTestMaxDeltaAfter
|
||
&& imageData[rowGTA + 1] - imageData[rowStart + 1] < arConf.edgeDetection.gradientTestMaxDeltaAfter
|
||
&& imageData[rowGTA + 2] - imageData[rowStart + 2] < arConf.edgeDetection.gradientTestMaxDeltaAfter
|
||
) {
|
||
// we do nothing (at this moment)
|
||
} else {
|
||
nonGradientPixelCount++;
|
||
}
|
||
} else {
|
||
nonGradientPixelCount++;
|
||
}
|
||
}
|
||
if (on) { // used to detect subtitles specifically
|
||
imageData[rowStart + 3] = GlDebugType.SubtitleThresholdOn;
|
||
isOnLetter = true;
|
||
letterSize++;
|
||
|
||
// bail on invalid letter sizes — this means we're seeing image.
|
||
// in this case, we do not need image confirmation step
|
||
if (letterSize > scanConf.maxValidLetter) {
|
||
updateImage(searchRow);
|
||
isBlank = false;
|
||
|
||
return;
|
||
}
|
||
}
|
||
|
||
rowStart += PIXEL_SIZE;
|
||
}
|
||
|
||
|
||
// we need to do this once more, otherwise imageSegmentCount can be 0 & bugs happen
|
||
if (isOnImage) {
|
||
if (imageSegmentSize < scanConf.maxValidLetter) {
|
||
potentialFadedLetterCount++;
|
||
|
||
// track potential letter alignment
|
||
imageSegmentAlignment += (rowStart - rowMid) * imageSegmentSize * PIXEL_SIZE_FRACTION;
|
||
imageSegmentAlignmentSamples += imageSegmentSize;
|
||
} else {
|
||
potentialFadedLetterCountInvalidated = true;
|
||
}
|
||
|
||
if (imageSegmentSize > arConf.edgeDetection.minEdgeSegmentSize) {
|
||
imageSegmentCount++;
|
||
imageWeightedSize += imageSegmentSize * imageSegmentSize; // longer segments should have bigger weight
|
||
segmentWeights += imageSegmentSize;
|
||
imageSegmentSize = 0;
|
||
}
|
||
}
|
||
|
||
|
||
if (!imageSegmentCount || isBlank) {
|
||
isBlank = true;
|
||
imageConfirmPass = false;
|
||
|
||
if (results.firstBlank === -1) {
|
||
results.firstBlank = searchRow;
|
||
}
|
||
results.lastBlank = searchRow;
|
||
|
||
resetSubtitlePass();
|
||
continue outerLoop;
|
||
}
|
||
|
||
const averageImageSegmentSize = segmentWeights > 0 ? imageWeightedSize / segmentWeights : 0;
|
||
const gradientDetectionFrequency = 1 - (nonGradientPixelCount / imageSize);
|
||
|
||
// That's probably a subtitle as well. If subs are fading in and out,
|
||
// then there's a good chance that they won't meet the "yes, this is a letter" threshold.
|
||
if (
|
||
!potentialFadedLetterCountInvalidated
|
||
&& potentialFadedLetterCount > scanConf.minDetections
|
||
// FOR SOME REASON THIS CONDITION CAUSES EVERYTHING TO HANG:
|
||
// && (imageSegmentAlignmentSamples && imageSegmentAlignment && Math.abs(imageSegmentAlignment / imageSegmentAlignmentSamples) > scanConf.maxPotentialSubtitleMisalignment)
|
||
&& averageImageSegmentSize < scanConf.maxValidLetter
|
||
) {
|
||
if (subtitleConfirmPass) {
|
||
if (results.firstSubtitle === -1) {
|
||
results.firstSubtitle = searchRow;
|
||
|
||
// if detecting subtitles only resets AR, we can return immediately
|
||
if (scanConf.subtitleCropMode !== AardSubtitleCropMode.CropSubtitles) {
|
||
break outerLoop;
|
||
}
|
||
}
|
||
results.lastSubtitle = searchRow;
|
||
isBlank = false;
|
||
|
||
resetSubtitlePass();
|
||
}
|
||
subtitleConfirmPass = true;
|
||
searchRow -= scanSpacing + (scanSpacing > 0 ? - 1 : 1);
|
||
continue outerLoop;
|
||
}
|
||
|
||
// If we are here, subtitles weren't confirmed
|
||
resetSubtitlePass();
|
||
|
||
|
||
// If we detect gradient, that's instant fail.
|
||
// We still save uncertain detection to firstImage, because iterative scan uses that
|
||
// in order to determine which region to scan further
|
||
if (gradientDetectionFrequency > arConf.edgeDetection.gradientThreshold) {
|
||
results.uncertain = true;
|
||
updateImage(searchRow);
|
||
break outerLoop;
|
||
}
|
||
|
||
// Cases which require confirmation
|
||
if (
|
||
imageSegmentCount > arConf.edgeDetection.maxEdgeSegments // we need to confirm if there's too many segments
|
||
|| averageImageSegmentSize < arConf.edgeDetection.averageEdgeThreshold // we also need to confirm if segments are too small
|
||
) {
|
||
if (imageConfirmPass) {
|
||
results.uncertain = true;
|
||
updateImage(searchRow - 1);
|
||
break outerLoop;
|
||
}
|
||
|
||
imageConfirmPass = true;
|
||
// imageConfirmPass must happen on the next row, but it's possible that we aren't
|
||
// checking row-by-row. Hence, we need to modify our scan a bit
|
||
searchRow -= scanSpacing + (scanSpacing > 0 ? - 1 : 1);
|
||
|
||
continue outerLoop;
|
||
}
|
||
|
||
if (averageImageSegmentSize > arConf.edgeDetection.averageEdgeThreshold || imageSize > imageThreshold) {
|
||
updateImage(searchRow);
|
||
break outerLoop;
|
||
}
|
||
|
||
// we can only reach this far if we detected image
|
||
if (imageConfirmPass) {
|
||
updateImage(searchRow - 1);
|
||
} else {
|
||
updateImage(searchRow);
|
||
}
|
||
break outerLoop;
|
||
} // end of outer loop
|
||
}
|
||
|
||
/**
|
||
* Tries to determine letterbox through subtitles
|
||
* @param imageData
|
||
* @param startRow
|
||
* @param endRow
|
||
* @param ROW_SIZE
|
||
* @param scanSpacing
|
||
* @param minDetections
|
||
* @param results
|
||
* @param ssrRegionName
|
||
* @returns
|
||
*/
|
||
private subtitleScanRegionIterative(
|
||
imageData: Uint8Array,
|
||
height: number,
|
||
startRow: number,
|
||
endRow: number,
|
||
scanSpacing: number,
|
||
minDetections: number,
|
||
results: AardTestResult_SubtitleRegion,
|
||
): boolean {
|
||
while (true) {
|
||
if (scanSpacing > -1 && scanSpacing < 1) {
|
||
break;
|
||
}
|
||
this.subtitleScanRegionLinear(
|
||
imageData, height,
|
||
startRow, endRow,
|
||
scanSpacing, minDetections, results
|
||
);
|
||
|
||
if (results.firstImage === -1) {
|
||
return false;
|
||
}
|
||
|
||
// We need to ensure small amount of overlap, in case we landed on a sus row
|
||
if (scanSpacing > 0) {
|
||
startRow = Math.max(results.firstImage - Math.floor(scanSpacing * 2), 0);
|
||
endRow = Math.min(results.firstImage + Math.floor(scanSpacing * 2), height - 1);
|
||
} else {
|
||
startRow = Math.min(results.firstImage + Math.floor(-scanSpacing * 2), height - 1);
|
||
endRow = Math.max(results.firstImage - Math.floor(-scanSpacing * 2), 0);
|
||
}
|
||
|
||
scanSpacing = scanSpacing / 2;
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
|
||
|
||
/**
|
||
* Updates aspect ratio if new aspect ratio is different enough from the old one
|
||
*/
|
||
private updateAspectRatio(ar: number, options?: {uncertainDetection?: boolean, forceReset?: boolean}) {
|
||
// Calculate difference between two ratios
|
||
|
||
// We need to detect updates even if subtitles are detected — we just don't trigger
|
||
// the actual aspect ratio change if everything is paused.
|
||
if (this.timers.pauseUntil > Date.now() && !options?.forceReset) {
|
||
return false;
|
||
}
|
||
|
||
const maxRatio = Math.max(ar, this.testResults.activeAspectRatio);
|
||
const diff = Math.abs(ar - this.testResults.activeAspectRatio);
|
||
|
||
if ((diff / maxRatio) > this.settings.active.aard.allowedArVariance || options?.forceReset) {
|
||
this.videoData.resizer.updateAr({
|
||
type: AspectRatioType.AutomaticUpdate,
|
||
ratio: ar,
|
||
offset: this.testResults.letterboxOffset,
|
||
variant: this.arVariant
|
||
});
|
||
this.testResults.activeAspectRatio = ar;
|
||
|
||
if (!options?.uncertainDetection) {
|
||
if (this.settings.active.aard.autoDisable.onFirstChange) {
|
||
this.status.autoDisabled = true;
|
||
}
|
||
if (this.settings.active.aard.autoDisable.ifNotChanged) {
|
||
this.timers.autoDisableAt = Date.now() + this.settings.active.aard.autoDisable.ifNotChangedTimeout;
|
||
}
|
||
}
|
||
|
||
this.testResults.aspectRatioUpdated = true;
|
||
}
|
||
}
|
||
|
||
/**
|
||
* Calculates video's current aspect ratio based on data in testResults.
|
||
* @returns
|
||
*/
|
||
private getAr(): number {
|
||
const fileAr = this.video.videoWidth / this.video.videoHeight;
|
||
const canvasAr = this.canvasStore.main.width / this.canvasStore.main.height;
|
||
|
||
const compensatedWidth = fileAr === canvasAr ? this.canvasStore.main.width : this.video.videoWidth * this.canvasStore.main.height / (this.video.videoHeight);
|
||
|
||
// console.log(`
|
||
// ———— ASPECT RATIO CALCULATION: —————
|
||
|
||
// canvas size: ${this.canvasStore.main.width} x ${this.canvasStore.main.height} (1:${this.canvasStore.main.width / this.canvasStore.main.height})
|
||
// file size: ${this.video.videoWidth} x ${this.video.videoHeight} (1:${this.video.videoWidth / this.video.videoHeight})
|
||
|
||
// compensated size: ${compensatedWidth} x ${this.canvasStore.main.height} (1:${compensatedWidth / this.canvasStore.main.height})
|
||
|
||
// letterbox height: ${this.testResults.letterboxWidth}
|
||
// net video height: ${this.canvasStore.main.height - (this.testResults.letterboxWidth * 2)}
|
||
|
||
// calculated aspect ratio -----
|
||
|
||
// ${compensatedWidth} ${compensatedWidth} ${compensatedWidth}
|
||
// ——————————————— = —————————————— = —————— = ${compensatedWidth / (this.canvasStore.main.height - (this.testResults.letterboxWidth * 2))}
|
||
// ${this.canvasStore.main.height} - 2 x ${this.testResults.letterboxWidth} ${this.canvasStore.main.height} - ${2 * this.testResults.letterboxWidth} ${this.canvasStore.main.height - (this.testResults.letterboxWidth * 2)}
|
||
// `);
|
||
|
||
if (this.testResults.letterboxOrientation === LetterboxOrientation.Pillarbox) {
|
||
const compensationFactor = compensatedWidth / this.canvasStore.main.width;
|
||
const pillarboxCompensated = (this.testResults.letterboxSize * 2 * compensationFactor);
|
||
|
||
return (compensatedWidth - pillarboxCompensated) / this.canvasStore.main.height;
|
||
} else {
|
||
const heightWithoutLetterbox = this.canvasStore.main.height - (this.testResults.letterboxSize * 2);
|
||
|
||
// TODO: set flag if subtitles are far enough from edge to avoid getting cropped
|
||
// if (this.testResults.subtitleDetected) {
|
||
// const hwlWithSubtitles = this.canvasStore.main.height - (this.testResults.letterboxSizeWithSubtitles * 2)
|
||
// const subtitleRatio = compensatedWidth / hwlWithSubtitles;
|
||
|
||
// }
|
||
return compensatedWidth / heightWithoutLetterbox;
|
||
}
|
||
}
|
||
|
||
//#endregion
|
||
|
||
}
|