diff --git a/src/csui/src/PlayerUIWindow.vue b/src/csui/src/PlayerUIWindow.vue index 7d5e9ee..8fa134e 100644 --- a/src/csui/src/PlayerUIWindow.vue +++ b/src/csui/src/PlayerUIWindow.vue @@ -385,7 +385,7 @@ export default { padding: 2rem; font-size: 1.5rem; - height: 4rem; + height: 6rem; border-bottom: 1px solid rgba(128, 128, 128, 0.5); border-top: 1px solid rgba(128, 128, 128, 0.5); diff --git a/src/csui/src/PlayerUiPanels/BaseExtensionSettings.vue b/src/csui/src/PlayerUiPanels/BaseExtensionSettings.vue index bf21f0a..e8b8518 100644 --- a/src/csui/src/PlayerUiPanels/BaseExtensionSettings.vue +++ b/src/csui/src/PlayerUiPanels/BaseExtensionSettings.vue @@ -101,7 +101,7 @@

Settings editor

-
+
Enable save button:
diff --git a/src/ext/lib/aard/Aard.ts b/src/ext/lib/aard/Aard.ts index 01e2ea4..41faf57 100644 --- a/src/ext/lib/aard/Aard.ts +++ b/src/ext/lib/aard/Aard.ts @@ -7,6 +7,7 @@ import Settings from '../Settings'; import { SiteSettings } from '../settings/SiteSettings'; import VideoData from '../video-data/VideoData'; import { AardDebugUi } from './AardDebugUi'; +import { AardTimer } from './AardTimers'; import { Corner } from './enums/corner.enum'; import { VideoPlaybackState } from './enums/video-playback-state.enum'; import { FallbackCanvas } from './gl/FallbackCanvas'; @@ -260,12 +261,14 @@ export class Aard { private fallbackReason: any; private canvasStore: AardCanvasStore; private testResults: AardTestResults; + private verticalTestResults: AardTestResults; private canvasSamples: AardDetectionSample; private forceFullRecheck: boolean = true; private debugConfig: any = {}; + private timer: AardTimer; //#endregion //#region getters @@ -301,6 +304,8 @@ export class Aard { // we can tick manually, for debugging this.logger.log('info', 'init', `[ArDetector::ctor] creating new ArDetector. arid: ${this.arid}`); + this.timer = new AardTimer() + this.init(); } @@ -403,7 +408,9 @@ export class Aard { */ startCheck() { if (!this.videoData.player) { - console.warn('Player not detected!') + console.warn('Player not detected!'); + console.log('--- video data: ---\n', this.videoData); + return; } if (this.siteSettings.data.enableAard[this.videoData.player.environment] === ExtensionMode.Enabled) { this.start(); @@ -424,6 +431,7 @@ export class Aard { // do full reset of test samples this.testResults = initAardTestResults(this.settings.active.arDetect); + this.verticalTestResults = initAardTestResults(this.settings.active.arDetect); if (this.animationFrame) { window.cancelAnimationFrame(this.animationFrame); @@ -437,8 +445,15 @@ export class Aard { * Runs autodetection ONCE. * If autodetection loop is running, this will also stop autodetection loop. */ - step() { + step(options?: {noCache?: boolean}) { this.stop(); + + if (options?.noCache) { + this.testResults = initAardTestResults(this.settings.active.arDetect); + this.verticalTestResults = initAardTestResults(this.settings.active.arDetect); + } + + this.main(); } @@ -500,7 +515,10 @@ export class Aard { */ private async main() { try { + this.timer.next(); + let imageData: Uint8Array; + this.timer.current.start = performance.now(); // We abuse a do-while loop to eat our cake (get early returns) // and have it, too (if we return early, we still execute code @@ -510,6 +528,7 @@ export class Aard { resolve => { try { this.canvasStore.main.drawVideoFrame(this.video); + this.timer.current.draw = performance.now() - this.timer.current.start; resolve(this.canvasStore.main.getImageData()); } catch (e) { if (e.name === 'SecurityError') { @@ -539,6 +558,7 @@ export class Aard { } } ); + this.timer.current.getImage = performance.now() - this.timer.current.start; // STEP 1: // Test if corners are black. If they're not, we can immediately quit the loop. @@ -547,6 +567,8 @@ export class Aard { this.settings.active.arDetect.canvasDimensions.sampleCanvas.width, this.settings.active.arDetect.canvasDimensions.sampleCanvas.height ); + this.timer.current.fastBlackLevel = performance.now() - this.timer.current.start; + if (this.testResults.notLetterbox) { // TODO: reset aspect ratio to "AR not applied" this.testResults.lastStage = 1; @@ -592,6 +614,7 @@ export class Aard { ); } } + this.timer.current.guardLine = performance.now() - this.timer.current.start; // guardLine is for both guardLine and imageLine checks // Both need to be checked if (! (this.testResults.imageLine.invalidated || this.testResults.guardLine.invalidated)) { @@ -626,6 +649,7 @@ export class Aard { if (this.testResults.notLetterbox) { // console.log('————not letterbox') console.warn('DETECTED NOT LETTERBOX! (resetting)') + this.timer.arChanged(); this.updateAspectRatio(this.defaultAr); break; } @@ -636,6 +660,7 @@ export class Aard { // console.warn('check finished:', JSON.parse(JSON.stringify(this.testResults)), JSON.parse(JSON.stringify(this.canvasSamples)), '\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n'); console.warn('ASPECT RATIO UNCERTAIN, GUARD LINE INVALIDATED (resetting)') + this.timer.arChanged(); this.updateAspectRatio(this.defaultAr); break; @@ -654,6 +679,7 @@ export class Aard { // except aspectRatioUpdated doesn't get set reliably, so we just call update every time, and update // if detected aspect ratio is different from the current aspect ratio // if (this.testResults.aspectRatioUpdated) { + // this.timer.arChanged(); this.updateAspectRatio(); // } @@ -662,6 +688,7 @@ export class Aard { if (this.canvasStore.debug) { // this.canvasStore.debug.drawBuffer(imageData); + this.timer.getAverage(); this.debugConfig?.debugUi?.updateTestResults(this.testResults); } } catch (e) { @@ -670,7 +697,6 @@ export class Aard { } } - private getVideoPlaybackState(): VideoPlaybackState { try { if (this.video.ended) { @@ -1266,12 +1292,14 @@ export class Aard { // bit more nicely visible (instead of hidden among spagheti) this.edgeScan(imageData, width, height); this.validateEdgeScan(imageData, width, height); + this.timer.current.edgeScan = performance.now() - this.timer.current.start; // TODO: _if gradient detection is enabled, then: this.sampleForGradient(imageData, width, height); + this.timer.current.gradient = performance.now() - this.timer.current.start; this.processScanResults(imageData, width, height); - + this.timer.current.scanResults = performance.now() - this.timer.current.start; } /** @@ -1435,6 +1463,9 @@ export class Aard { const slopeTestSample = this.settings.active.arDetect.edgeDetection.slopeTestWidth * 4; while (i < this.canvasSamples.top.length) { + // if (this.canvasSamples.top[i] < 0) { + // continue; + // } // calculate row offset: row = (this.canvasSamples.top[i + 1] - 1) * width * 4; xs = row + this.canvasSamples.top[i] - slopeTestSample; @@ -1460,6 +1491,10 @@ export class Aard { i = 0; let i1 = 0; while (i < this.canvasSamples.bottom.length) { + // if (this.canvasSamples.bottom[i] < 0) { + // continue; + // } + // calculate row offset: i1 = i + 1; row = (this.canvasSamples.bottom[i1] + 1) * width * 4; @@ -1506,6 +1541,10 @@ export class Aard { upperEdgeCheck: for (let i = 1; i < this.canvasSamples.top.length; i += 2) { + if (this.canvasSamples.top[i] < 0) { + continue; + } + pixelOffset = this.canvasSamples.top[i] * realWidth + this.canvasSamples.top[i - 1] * 4; lastSubpixel = imageData[pixelOffset] > imageData[pixelOffset + 1] ? imageData[pixelOffset] : imageData[pixelOffset + 1]; @@ -1548,6 +1587,9 @@ export class Aard { lowerEdgeCheck: for (let i = 1; i < this.canvasSamples.bottom.length; i += 2) { + if (this.canvasSamples.bottom[i] < 0) { + continue; + } pixelOffset = (height - this.canvasSamples.bottom[i]) * realWidth + this.canvasSamples.bottom[i - 1] * 4; lastSubpixel = imageData[pixelOffset] > imageData[pixelOffset + 1] ? imageData[pixelOffset] : imageData[pixelOffset + 1]; @@ -1586,7 +1628,6 @@ export class Aard { if (lastSubpixel - firstSubpixel > this.settings.active.arDetect.edgeDetection.gradientTestMinDelta) { this.canvasSamples.bottom[i] = -1; } - } } @@ -1943,10 +1984,21 @@ export class Aard { return; } if (maxOffset > 2) { - this.testResults.imageLine.top = this.testResults.aspectRatioCheck.topCandidate === Infinity ? -1 : this.testResults.aspectRatioCheck.topCandidate; - this.testResults.imageLine.bottom = this.testResults.aspectRatioCheck.bottomCandidate === Infinity ? -1 : this.testResults.aspectRatioCheck.bottomCandidate; - this.testResults.guardLine.top = Math.max(this.testResults.imageLine.top - 2, 0); - this.testResults.guardLine.bottom = Math.min(this.testResults.imageLine.bottom + 2, this.canvasStore.main.height - 1); + if (this.testResults.aspectRatioCheck.topCandidate === Infinity) { + this.testResults.imageLine.top = -1; + this.testResults.guardLine.top = -1; + } else { + this.testResults.imageLine.top = this.testResults.aspectRatioCheck.topCandidate = this.testResults.aspectRatioCheck.topCandidate; + this.testResults.guardLine.top = Math.max(this.testResults.imageLine.top - 2, 0); + } + + if (this.testResults.aspectRatioCheck.bottomCandidate === Infinity) { + this.testResults.imageLine.bottom = -1; + this.testResults.guardLine.bottom = -1; + } else { + this.testResults.imageLine.bottom = this.testResults.aspectRatioCheck.bottomCandidate; + this.testResults.guardLine.bottom = Math.min(this.testResults.imageLine.bottom + 2, this.canvasStore.main.height - 1); + } } this.testResults.aspectRatioUncertain = false; diff --git a/src/ext/lib/aard/AardDebugUi.ts b/src/ext/lib/aard/AardDebugUi.ts index 100fb5c..5a50fe8 100644 --- a/src/ext/lib/aard/AardDebugUi.ts +++ b/src/ext/lib/aard/AardDebugUi.ts @@ -1,3 +1,5 @@ +import { AardPerformanceData } from './AardTimers'; + export class AardDebugUi { aard: any; @@ -24,7 +26,38 @@ export class AardDebugUi { position: fixed; top: 0; left: 0; width: 100vw; height: 100dvh; display: flex; flex-direction: column; pointer-events: none; z-index: 9999; font-size: 16px; font-family: 'Overpass Mono', monospace; ">
-

Aaard debug overlay

+

Aard debug overlay

+ + + +
+
+
+
+
@@ -64,7 +97,8 @@ export class AardDebugUi { - + +
@@ -93,6 +127,7 @@ export class AardDebugUi { document.getElementById('uw-aard-debug-ui_disable-stop-on-change').onclick = () => this.changePauseOnCheck(false); document.getElementById('uw-aard-debug-ui_resume-video').onclick = () => this.resumeVideo(); document.getElementById('uw-aard-debug-ui_enable-step').onclick = () => this.aard.step(); + document.getElementById('uw-aard-debug-ui_enable-step-nocache').onclick = () => this.aard.step({noCache: true}); document.getElementById('uw-aard-debug-ui_close-overlay').onclick = () => (this.aard as any).hideDebugCanvas(); } @@ -119,7 +154,181 @@ export class AardDebugUi { this.aard.start(); } + private updatePerformanceResults() { + const previewDiv = document.getElementById('uw-aard-debug_performance-mini'); + const popupDiv = document.getElementById('uw-aard-debug_performance-popup'); + + const previewContent = ` +
+
Current:
+
${this.generateMiniGraphBar(this.aard.timer.current)}
+
+
+
Average:
+
${this.generateMiniGraphBar(this.aard.timer.average)}
+
+
+
Last chg.:
${this.generateMiniGraphBar(this.aard.timer.lastChange)}
+
+ `; + + const popupContent = ` +

Detailed performance analysis:

+
+
+ Raw times (cumulative; -1 = test was skipped):
+ draw ${this.aard.timer.current.draw.toFixed(2)}ms + get data ${this.aard.timer.current.getImage.toFixed(2)}ms + black lv. ${this.aard.timer.current.fastBlackLevel.toFixed(2)}ms + guard/image ${this.aard.timer.current.guardLine.toFixed(3)}ms + edge ${this.aard.timer.current.edgeScan.toFixed(2)}ms + gradient ${this.aard.timer.current.gradient.toFixed(3)}ms + post ${this.aard.timer.current.scanResults.toFixed(2)}ms +
+
Stage times (not cumulative):
+
+
Current:
+
${this.generateMiniGraphBar(this.aard.timer.current, true)}
+
+ +
+ Raw times (cumulative; -1 = test was skipped):
+ draw ${this.aard.timer.average.draw.toFixed(2)}ms + get data ${this.aard.timer.average.getImage.toFixed(2)}ms + black lv. ${this.aard.timer.average.fastBlackLevel.toFixed(2)}ms + guard/image ${this.aard.timer.average.guardLine.toFixed(3)}ms + edge ${this.aard.timer.average.edgeScan.toFixed(2)}ms + gradient ${this.aard.timer.average.gradient.toFixed(3)}ms + post ${this.aard.timer.average.scanResults.toFixed(2)}ms +
+
Stage times (not cumulative):
+
+
Average:
+
${this.generateMiniGraphBar(this.aard.timer.average, true)}
+
+ +
+
Last change:
+
${this.generateMiniGraphBar(this.aard.timer.lastChange, true)}
+
+
+ ` + + previewDiv.innerHTML = previewContent; + popupDiv.innerHTML = popupContent; + } + + private getBarLabel(width: number, leftOffset: number, topOffset: number, label: string, detailed: boolean, extraStyles?: string) { + if (!detailed) { + return ''; + } + + let offsets: string; + let text: string = ''; + + if (leftOffset + width < 80) { + // at the end of the bar + offsets = `left: ${leftOffset + width}%;`; + } else { + if (width < 15 && leftOffset < 100) { + // before the bar + offsets = `right: ${100 - leftOffset}%;`; + text = 'color: #fff;'; + } else { + // inside the bar, aligned to right + offsets = `right: ${Math.max(100 - (leftOffset + width), 0)}%;` + } + } + + return ` +
+ ${label} +
+ `; + } + + private generateMiniGraphBar(perf: AardPerformanceData, detailed: boolean = false) { + if (!perf) { + return ` + n/a + `; + } + + let total = 0; + + const draw = Math.max(perf.draw, 0); + total += draw; + + const getImageStart = draw; + const getImage = Math.max(perf.getImage - total, 0); + total += getImage; + + const fastBlackLevelStart = getImageStart + getImage; + const fastBlackLevel = Math.max(perf.fastBlackLevel - total, 0); + total += fastBlackLevel; + + const guardLineStart = fastBlackLevelStart + fastBlackLevel; + const guardLine = Math.max(perf.guardLine - total, 0); + total += guardLine; + + const edgeScanStart = guardLineStart + guardLine; + const edgeScan = Math.max(perf.edgeScan - total, 0); + total += edgeScan; + + const gradientStart = edgeScanStart + edgeScan; + const gradient = Math.max(perf.gradient - total, 0); + total += gradient; + + const scanResultsStart = gradientStart + gradient; + const scanResults = Math.max(perf.scanResults - total, 0); + total += scanResults; + + return ` +
+ ${detailed ? '' : `${total.toFixed()} ms`} +
+ ${detailed ? `
` : ''} + +
+ ${this.getBarLabel(draw, 0, 2, `draw: ${draw.toFixed(2)} ms`, detailed)} + +
+ ${this.getBarLabel(getImage, getImageStart, 14, `get data: ${getImage.toFixed(2)} ms`, detailed)} + +
+ ${this.getBarLabel(fastBlackLevel, fastBlackLevelStart, 26, `black level: ${fastBlackLevel.toFixed(2)} ms`, detailed)}; + +
+ ${this.getBarLabel(guardLine, guardLineStart, 38, `guard/image line: ${guardLine.toFixed(2)} ms`, detailed)} + +
+ ${this.getBarLabel(edgeScan, edgeScanStart, 50, `edge scan (/w validation): ${edgeScan.toFixed(2)} ms`, detailed)} + +
+ ${this.getBarLabel(gradient, gradientStart, 62, `gradient: ${gradient.toFixed(2)} ms`, detailed)} + +
+ ${this.getBarLabel(scanResults, scanResultsStart, 74, `scan results processing: ${scanResults.toFixed(2)} ms`, detailed)} + + ${this.getBarLabel(0, scanResults + scanResultsStart, 88, `total: ${total.toFixed(2)} ms`, detailed, 'color: #fff;')} + + +
60fps
+
30fps
+
+ `; + } + updateTestResults(testResults) { + this.updatePerformanceResults(); + if (testResults.aspectRatioUpdated && this.pauseOnArCheck) { (this.aard as any).video.pause(); this.aard.stop(); diff --git a/src/ext/lib/aard/AardTimers.ts b/src/ext/lib/aard/AardTimers.ts new file mode 100644 index 0000000..b52b5cd --- /dev/null +++ b/src/ext/lib/aard/AardTimers.ts @@ -0,0 +1,110 @@ +export interface AardPerformanceData { + start: number; + draw: number; + getImage: number; + + fastBlackLevel: number; + guardLine: number; // actually times both guard line and image line checks + edgeScan: number; // includes validation step + gradient: number; + scanResults: number; +} + + +export class AardTimer { + private currentIndex: number = -1; + + private aardPerformanceDataBuffer: AardPerformanceData[]; + + current: AardPerformanceData; + previous: AardPerformanceData; + average: AardPerformanceData; + lastChange: AardPerformanceData; + + + constructor() { + this.aardPerformanceDataBuffer = new Array(16).fill(this.getEmptyMeasurement()); + this.current = this.aardPerformanceDataBuffer[0]; + this.previous = undefined; + this.lastChange = this.getEmptyMeasurement(); + this.average = this.getEmptyMeasurement(); + } + + private getEmptyMeasurement(): AardPerformanceData { + return { + start: -1, + draw: -1, + getImage: -1, + fastBlackLevel: -1, + guardLine: -1, + edgeScan: -1, + gradient: -1, + scanResults: -1 + } + }; + + private clearMeasurement(index: number) { + this.aardPerformanceDataBuffer[index].draw = -1; + this.aardPerformanceDataBuffer[index].getImage = -1; + this.aardPerformanceDataBuffer[index].fastBlackLevel = -1; + this.aardPerformanceDataBuffer[index].guardLine = -1; + this.aardPerformanceDataBuffer[index].edgeScan = -1; + this.aardPerformanceDataBuffer[index].gradient = -1; + this.aardPerformanceDataBuffer[index].scanResults = -1; + } + + next() { + // go to next buffer position; + this.currentIndex = (this.currentIndex + 1) % this.aardPerformanceDataBuffer.length; + this.previous = this.current; + this.clearMeasurement(this.currentIndex); + + // TODO: reset values + this.current = this.aardPerformanceDataBuffer[this.currentIndex]; + } + + arChanged() { // copy numbers over + this.lastChange.draw = this.current.draw; + this.lastChange.getImage = this.current.getImage; + this.lastChange.fastBlackLevel = this.current.fastBlackLevel; + this.lastChange.guardLine = this.current.guardLine; + this.lastChange.edgeScan = this.current.edgeScan; + this.lastChange.gradient = this.current.gradient; + this.lastChange.scanResults = this.current.scanResults; + } + + getAverage() { + for (let i = 0; i < this.aardPerformanceDataBuffer.length; i++) { + const sample = this.aardPerformanceDataBuffer[i]; + if (sample.draw !== -1) { + this.average.draw += sample.draw; + } + if (sample.getImage !== -1) { + this.average.getImage += sample.getImage; + } + if (sample.fastBlackLevel !== -1) { + this.average.fastBlackLevel += sample.fastBlackLevel; + } + if (sample.guardLine !== -1) { + this.average.guardLine += sample.guardLine; + } + if (sample.edgeScan !== -1) { + this.average.edgeScan += sample.edgeScan; + } + if (sample.gradient !== -1) { + this.average.gradient += sample.gradient; + } + if (sample.scanResults !== -1) { + this.average.scanResults += sample.scanResults; + } + } + + this.average.draw /= this.aardPerformanceDataBuffer.length; + this.average.getImage /= this.aardPerformanceDataBuffer.length; + this.average.fastBlackLevel /= this.aardPerformanceDataBuffer.length; + this.average.guardLine /= this.aardPerformanceDataBuffer.length; + this.average.edgeScan /= this.aardPerformanceDataBuffer.length; + this.average.gradient /= this.aardPerformanceDataBuffer.length; + this.average.scanResults /= this.aardPerformanceDataBuffer.length; + } +}