From 8cef734926323c140322efcda257e61145779de4 Mon Sep 17 00:00:00 2001 From: Tamius Han Date: Fri, 3 May 2019 00:49:33 +0200 Subject: [PATCH] Autodetection/black frame check: use more strict/lax rules depending on color variance --- src/ext/conf/ExtensionConf.js | 8 ++- src/ext/lib/ar-detect/ArDetector.js | 100 ++++++++++++++++++++++++++-- 2 files changed, 101 insertions(+), 7 deletions(-) diff --git a/src/ext/conf/ExtensionConf.js b/src/ext/conf/ExtensionConf.js index c9dfd4b..c41885e 100644 --- a/src/ext/conf/ExtensionConf.js +++ b/src/ext/conf/ExtensionConf.js @@ -45,7 +45,13 @@ var ExtensionConf = { // samplingInterval: 10, // we sample at columns at (width/this) * [ 1 .. this - 1] blackframe: { - cumulativeThreshold: 2560, // if we add values of all pixels together and get more than this, the frame is bright enough. + sufficientColorVariance: 0.05, // calculate difference between average intensity and pixel, for every pixel for every color + // component. Average intensity is normalized to where 0 is black and 1 is biggest value for + // that component. If sum of differences between normalized average intensity and normalized + // component varies more than this % between color components, we can afford to use less strict + // cummulative treshold. + cumulativeTresholdLax: 1600, + cumulativeThresholdStrict: 2560,// if we add values of all pixels together and get more than this, the frame is bright enough. // (note: blackframe is 16x9 px -> 144px total. cumulative threshold can be reached fast) blackPixelsCondition: 0.6, // How much pixels must be black (1 all, 0 none) before we consider frame as black. Takes // precedence over cumulative threshold: if blackPixelsCondition is met, the frame is dark diff --git a/src/ext/lib/ar-detect/ArDetector.js b/src/ext/lib/ar-detect/ArDetector.js index 8f2f059..c5a39da 100644 --- a/src/ext/lib/ar-detect/ArDetector.js +++ b/src/ext/lib/ar-detect/ArDetector.js @@ -515,14 +515,14 @@ class ArDetector { frameCheck(){ if(! this.video){ - if(Debug.debug || Debug.warnings_critical) + if(Debug.debug || Debug.warnings_critical) { console.log("[ArDetect::frameCheck] Video went missing. Destroying current instance of videoData.") + } this.conf.destroy(); return; } if (!this.blackframeContext) { - console.log("BLACKFRAME CONTEXT IS NOT DEFINED", this.blackframeContext); this.init(); } @@ -581,12 +581,16 @@ class ArDetector { } const bfanalysis = this.blackframeTest(); - if (bfanalysis.isBlack) { // we don't do any corrections on frames confirmed black + if (Debug.debug && Debug.arDetect) { + console.log("%c[ArDetect::frameCheck] Black frame analysis suggests this frame is black or too dark. Doing nothing,", "color: #fa3", bfanalysis); + } return; } + + // if we are in fallback mode, then frame has already been drawn to the main canvas. // if we are in normal mode though, the frame has yet to be drawn if (!this.fallbackMode) { @@ -599,11 +603,14 @@ class ArDetector { // da je letterbox izginil. // If we don't detect letterbox, we reset aspect ratio to aspect ratio of the video file. The aspect ratio could // have been corrected manually. It's also possible that letterbox (that was there before) disappeared. - console.log("FAST LETTERBOX PRESENCE TEST FAILED, CALLING RESET") this.conf.resizer.reset({type: AspectRatio.Automatic, ratio: null}); this.guardLine.reset(); this.noLetterboxCanvasReset = true; + + if (Debug.debug && Debug.arDetect) { + console.log("%c[ArDetect::frameCheck] Letterbox not detected in fast test. Letterbox is either gone or we manually corrected aspect ratio. Nothing will be done.", "color: #fa3"); + } return; } @@ -697,6 +704,11 @@ class ArDetector { if (edgePost.status !== EdgeStatus.AR_KNOWN){ // rob ni bil zaznan, zato ne naredimo ničesar. // no edge was detected. Let's leave things as they were + + if (Debug.debug && Debug.arDetect) { + console.log("%c[ArDetect::frameCheck] Edge wasn't detected with findBars", "color: #fa3", edgePost, "EdgeStatus.AR_KNOWN:", EdgeStatus.AR_KNOWN); + } + return; } @@ -764,6 +776,12 @@ class ArDetector { const rows = this.blackframeCanvas.height; const cols = this.blackframeCanvas.width; + const pixels = rows * cols; + let cumulative_r = 0, cumulative_g = 0, cumulative_b = 0; + let max_r = 0, max_g = 0, max_b = 0; + let avg_r, avg_g, avg_b; + let var_r = 0, var_g = 0, var_b = 0; + let pixelMax = 0; let cumulativeValue = 0; let blackPixelCount = 0; @@ -777,17 +795,26 @@ class ArDetector { let r, c; + for (let i = 0; i < bfImageData.length; i+= 4) { pixelMax = Math.max(bfImageData[i], bfImageData[i+1], bfImageData[i+2]); + bfImageData[i+3] = pixelMax; if (pixelMax < this.blackLevel) { this.blackLevel = pixelMax; blackPixelCount++; } else { cumulativeValue += pixelMax; + cumulative_r += bfImageData[i]; + cumulative_g += bfImageData[i+1]; + cumulative_b += bfImageData[i+2]; + + max_r = max_r > bfImageData[i] ? max_r : bfImageData[i]; + max_g = max_g > bfImageData[i+1] ? max_g : bfImageData[i+1]; + max_b = max_b > bfImageData[i+2] ? max_b : bfImageData[i+2]; } - r = Math.floor(i/rows); + r = ~~(i/rows); c = i % cols; if (pixelMax > rowMax[r]) { @@ -798,8 +825,69 @@ class ArDetector { } } + max_r = 1 / (max_r || 1); + max_g = 1 / (max_g || 1); + max_b = 1 / (max_b || 1); + + const imagePixels = pixels - blackPixelCount; + // calculate averages and normalize them + avg_r = (cumulative_r / imagePixels) * max_r; + avg_g = (cumulative_g / imagePixels) * max_g; + avg_b = (cumulative_b / imagePixels) * max_b; + + // second pass for color variance + for (let i = 0; i < bfImageData.length; i+= 4) { + if (bfImageData[i+3] >= this.blackLevel) { + var_r += Math.abs(avg_r - bfImageData[i] * max_r); + var_g += Math.abs(avg_g - bfImageData[i+1] * max_g); + var_b += Math.abs(avg_b - bfImageData[i+1] * max_b); + } + } + + const hasSufficientVariance = Math.abs(var_r - var_g) / Math.max(var_r, var_g, 1) > this.settings.active.arDetect.blackframe.sufficientColorVariance + || Math.abs(var_r - var_b) / Math.max(var_r, var_b, 1) > this.settings.active.arDetect.blackframe.sufficientColorVariance + || Math.abs(var_b - var_g) / Math.max(var_b, var_g, 1) > this.settings.active.arDetect.blackframe.sufficientColorVariance + + let isBlack = (blackPixelCount/(cols * rows) > this.settings.active.arDetect.blackframe.blackPixelsCondition); + + if (! isBlack) { + if (hasSufficientVariance) { + isBlack = cumulativeValue < this.settings.active.arDetect.blackframe.cumulativeThresholdLax; + } else { + isBlack = cumulativeValue < this.settings.active.arDetect.blackframe.cumulativeThresholdStrict; + } + } + + if (Debug.debug) { + return { + isBlack: isBlack, + blackPixelCount: blackPixelCount, + blackPixelRatio: (blackPixelCount/(cols * rows)), + cumulativeValue: cumulativeValue, + hasSufficientVariance: hasSufficientVariance, + variances: { + raw: { + r: var_r, g: var_g, b: var_b + }, + relative: { + rg: Math.abs(var_r - var_g) / Math.max(var_r, var_g, 1), + rb: Math.abs(var_r - var_b) / Math.max(var_r, var_b, 1), + gb: Math.abs(var_b - var_g) / Math.max(var_b, var_g, 1), + }, + relativePercent: { + rg: Math.abs(var_r - var_g) / Math.max(var_r, var_g, 1) / this.settings.active.arDetect.blackframe.sufficientColorVariance, + rb: Math.abs(var_r - var_b) / Math.max(var_r, var_b, 1) / this.settings.active.arDetect.blackframe.sufficientColorVariance, + gb: Math.abs(var_b - var_g) / Math.max(var_b, var_g, 1) / this.settings.active.arDetect.blackframe.sufficientColorVariance, + }, + varianceLimit: this.settings.active.arDetect.blackframe.sufficientColorVariance, + }, + cumulativeValuePercent: cumulativeValue / (hasSufficientVariance ? this.settings.active.arDetect.blackframe.cumulativeThresholdLax : this.settings.active.arDetect.blackframe.cumulativeThresholdStrict), + rowMax: rowMax, + colMax: colMax, + }; + } return { - isBlack: (blackPixelCount/(cols * rows) > this.settings.active.arDetect.blackframe.blackPixelsCondition) ? true : cumulativeValue < this.settings.active.arDetect.blackframe.cumulativeThreshold, + isBlack: isBlack, rowMax: rowMax, colMax: colMax, };