tamius-han/ultrawidify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework blackframe to not use its own drawImage call

Browse Source
This commit is contained in:
Tamius Han 2022-07-13 00:22:16 +02:00
parent 1556506bd7
commit a09ea26b0e
1 changed files with 53 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
src/ext/lib/ar-detect/ArDetector.ts
View File

@ -198,20 +198,20 @@ class ArDetector {
if (this.canvas) { if (this.canvas) {
this.canvas.remove(); this.canvas.remove();
} }
if (this.blackframeCanvas) { // if (this.blackframeCanvas) {
this.blackframeCanvas.remove(); // this.blackframeCanvas.remove();
} // }
// things to note: we'll be keeping canvas in memory only. // things to note: we'll be keeping canvas in memory only.
this.canvas = document.createElement("canvas"); this.canvas = document.createElement("canvas");
this.canvas.width = cwidth; this.canvas.width = cwidth;
this.canvas.height = cheight; this.canvas.height = cheight;
this.blackframeCanvas = document.createElement("canvas"); // this.blackframeCanvas = document.createElement("canvas");
this.blackframeCanvas.width = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.width; // this.blackframeCanvas.width = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.width;
this.blackframeCanvas.height = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.height; // this.blackframeCanvas.height = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.height;
this.context = this.canvas.getContext("2d"); this.context = this.canvas.getContext("2d");
this.blackframeContext = this.blackframeCanvas.getContext("2d"); // this.blackframeContext = this.blackframeCanvas.getContext("2d");
// //
@ -836,15 +836,7 @@ class ArDetector {
let sampleCols = this.sampleCols.slice(0); let sampleCols = this.sampleCols.slice(0);
const bfAnalysis = await this.blackframeTest();
if (bfAnalysis.isBlack) {
// we don't do any corrections on frames confirmed black
this.logger.log('info', 'arDetect_verbose', `%c[ArDetect::frameCheck] Black frame analysis suggests this frame is black or too dark. Doing nothing.`, "color: #fa3", bfAnalysis);
timerResults.blackFrameDrawTime = bfAnalysis.processingTime.blackFrameDrawTime;
timerResults.blackFrameProcessTime = bfAnalysis.processingTime.blackFrameProcessTime;
this.addPerformanceMeasurement(timerResults);
return;
}
let startTime = performance.now(); let startTime = performance.now();
await new Promise<void>( await new Promise<void>(
@ -856,6 +848,16 @@ class ArDetector {
const imageData = this.context.getImageData(0, 0, this.canvas.width, this.canvas.height).data; const imageData = this.context.getImageData(0, 0, this.canvas.width, this.canvas.height).data;
timerResults.imageDrawTime = performance.now() - startTime; timerResults.imageDrawTime = performance.now() - startTime;
const bfAnalysis = await this.blackframeTest(imageData);
if (bfAnalysis.isBlack) {
// we don't do any corrections on frames confirmed black
this.logger.log('info', 'arDetect_verbose', `%c[ArDetect::frameCheck] Black frame analysis suggests this frame is black or too dark. Doing nothing.`, "color: #fa3", bfAnalysis);
timerResults.blackFrameDrawTime = bfAnalysis.processingTime.blackFrameDrawTime;
timerResults.blackFrameProcessTime = bfAnalysis.processingTime.blackFrameProcessTime;
this.addPerformanceMeasurement(timerResults);
return;
}
startTime = performance.now(); startTime = performance.now();
const fastLetterboxTestRes = this.fastLetterboxPresenceTest(imageData, sampleCols); const fastLetterboxTestRes = this.fastLetterboxPresenceTest(imageData, sampleCols);
timerResults.fastLetterboxTime = performance.now() - startTime; timerResults.fastLetterboxTime = performance.now() - startTime;
@ -969,7 +971,7 @@ class ArDetector {
this.blackLevel = this.settings.active.arDetect.blackbar.blackLevel; this.blackLevel = this.settings.active.arDetect.blackbar.blackLevel;
} }
async blackframeTest() { async blackframeTest(imageData) {
if (this.blackLevel === undefined) { if (this.blackLevel === undefined) {
this.logger.log('info', 'arDetect_verbose', "[ArDetect::blackframeTest] black level undefined, resetting"); this.logger.log('info', 'arDetect_verbose', "[ArDetect::blackframeTest] black level undefined, resetting");
this.resetBlackLevel(); this.resetBlackLevel();
@ -980,15 +982,19 @@ class ArDetector {
*/ */
const bfDrawStartTime = performance.now(); const bfDrawStartTime = performance.now();
await new Promise<void>( // await new Promise<void>(
resolve => { // resolve => {
this.blackframeContext.drawImage(this.video, 0, 0, this.blackframeCanvas.width, this.blackframeCanvas.height); // this.blackframeContext.drawImage(this.video, 0, 0, this.blackframeCanvas.width, this.blackframeCanvas.height);
resolve(); // resolve();
} // }
); // );
const rows = this.blackframeCanvas.height; // const rows = this.blackframeCanvas.height;
const cols = this.blackframeCanvas.width; // const cols = this.blackframeCanvas.width;
const bfImageData = this.blackframeContext.getImageData(0, 0, cols, rows).data; // const bfImageData = this.blackframeContext.getImageData(0, 0, cols, rows).data;
const rows = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.width;
const cols = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.height;
const samples = rows * cols;
const blackFrameDrawTime = performance.now() - bfDrawStartTime; const blackFrameDrawTime = performance.now() - bfDrawStartTime;
const bfProcessStartTime = performance.now(); const bfProcessStartTime = performance.now();
@ -1004,6 +1010,13 @@ class ArDetector {
let blackPixelCount = 0; let blackPixelCount = 0;
const blackThreshold = this.blackLevel + this.settings.active.arDetect.blackbar.frameThreshold; const blackThreshold = this.blackLevel + this.settings.active.arDetect.blackbar.frameThreshold;
const actualPixels = imageData.length / 4;
// generate some random points that we'll use for sampling black frame.
const sampleArray = new Array(samples);
for (let i = 0; i < samples; i++) {
sampleArray[i] = Math.round(Math.random() * actualPixels);
}
// we do some recon for letterbox and pillarbox. While this can't determine whether letterbox/pillarbox exists // we do some recon for letterbox and pillarbox. While this can't determine whether letterbox/pillarbox exists
// with sufficient level of certainty due to small sample resolution, it can still give us some hints for later // with sufficient level of certainty due to small sample resolution, it can still give us some hints for later
@ -1012,10 +1025,9 @@ class ArDetector {
let r: number, c: number; let r: number, c: number;
for (const i of sampleArray) {
for (let i = 0; i < bfImageData.length; i+= 4) { pixelMax = Math.max(imageData[i], imageData[i+1], imageData[i+2]);
pixelMax = Math.max(bfImageData[i], bfImageData[i+1], bfImageData[i+2]); imageData[i+3] = pixelMax;
bfImageData[i+3] = pixelMax;
if (pixelMax < blackThreshold) { if (pixelMax < blackThreshold) {
if (pixelMax < this.blackLevel) { if (pixelMax < this.blackLevel) {
@ -1024,13 +1036,13 @@ class ArDetector {
blackPixelCount++; blackPixelCount++;
} else { } else {
cumulativeValue += pixelMax; cumulativeValue += pixelMax;
cumulative_r += bfImageData[i]; cumulative_r += imageData[i];
cumulative_g += bfImageData[i+1]; cumulative_g += imageData[i+1];
cumulative_b += bfImageData[i+2]; cumulative_b += imageData[i+2];
max_r = max_r > bfImageData[i] ? max_r : bfImageData[i]; max_r = max_r > imageData[i] ? max_r : imageData[i];
max_g = max_g > bfImageData[i+1] ? max_g : bfImageData[i+1]; max_g = max_g > imageData[i+1] ? max_g : imageData[i+1];
max_b = max_b > bfImageData[i+2] ? max_b : bfImageData[i+2]; max_b = max_b > imageData[i+2] ? max_b : imageData[i+2];
} }
r = ~~(i/rows); r = ~~(i/rows);
@ -1055,11 +1067,11 @@ class ArDetector {
avg_b = (cumulative_b / imagePixels) * max_b; avg_b = (cumulative_b / imagePixels) * max_b;
// second pass for color variance // second pass for color variance
for (let i = 0; i < bfImageData.length; i+= 4) { for (let i = 0; i < imageData.length; i+= 4) {
if (bfImageData[i+3] >= this.blackLevel) { if (imageData[i+3] >= this.blackLevel) {
var_r += Math.abs(avg_r - bfImageData[i] * max_r); var_r += Math.abs(avg_r - imageData[i] * max_r);
var_g += Math.abs(avg_g - bfImageData[i+1] * max_g); var_g += Math.abs(avg_g - imageData[i+1] * max_g);
var_b += Math.abs(avg_b - bfImageData[i+1] * max_b); var_b += Math.abs(avg_b - imageData[i+1] * max_b);
} }
} }