ultrawidify/src/ext/lib/ar-detect/AardGl.js

662 lines
23 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

import Debug from '../../conf/Debug';
import EdgeDetect from './edge-detect/EdgeDetect';
import EdgeStatus from './edge-detect/enums/EdgeStatusEnum';
import EdgeDetectPrimaryDirection from './edge-detect/enums/EdgeDetectPrimaryDirectionEnum';
import EdgeDetectQuality from './edge-detect/enums/EdgeDetectQualityEnum';
import GuardLine from './GuardLine';
import DebugCanvas from './DebugCanvas';
import VideoAlignment from '../../../common/enums/video-alignment.enum';
import AspectRatio from '../../../common/enums/aspect-ratio.enum';
import { generateHorizontalAdder } from './gllib/shader-generators/HorizontalAdderGenerator';
import { getBasicVertexShader } from './gllib/shaders/vertex-shader';
import { sleep } from '../Util';
/**
* AardGl: Hardware accelerated aspect ratio detection script, based on WebGL
*/
class AardGl {
constructor(videoData){
this.logger = videoData.logger;
this.conf = videoData;
this.video = videoData.video;
this.settings = videoData.settings;
this.setupTimer = null;
this.sampleCols = [];
this.canFallback = true;
this.fallbackMode = false;
this.blackLevel = this.settings.active.aard.blackbar.blackLevel;
this.arid = (Math.random()*100).toFixed();
// ar detector starts in this state. running main() sets both to false
this._halted = true;
this._exited = true;
// we can tick manually, for debugging
this._manualTicks = false;
this._nextTick = false;
this.canDoFallbackMode = false;
this.logger.log('info', 'init', `[AardGl::ctor] creating new AardGl. arid: ${this.arid}`);
}
/**
*
* HELPER FUNCTIONS
*
*/
//#region helpers
canTriggerFrameCheck(lastFrameCheckStartTime) {
if (this._paused) {
return false;
}
if (this.video.ended || this.video.paused){
// we slow down if ended or pausing. Detecting is pointless.
// we don't stop outright in case seeking happens during pause/after video was
// ended and video gets into 'playing' state again
return Date.now() - lastFrameCheckStartTime > this.settings.active.aard.timers.paused;
}
if (this.video.error){
// če je video pavziran, še vedno skušamo zaznati razmerje stranic - ampak bolj poredko.
// if the video is paused, we still do autodetection. We just do it less often.
return Date.now() - lastFrameCheckStartTime > this.settings.active.aard.timers.error;
}
return Date.now() - lastFrameCheckStartTime > this.settings.active.aard.timers.playing;
}
isRunning(){
return ! (this._halted || this._paused || this._exited);
}
scheduleInitRestart(timeout, force_reset){
if(! timeout){
timeout = 100;
}
// don't allow more than 1 instance
if(this.setupTimer){
clearTimeout(this.setupTimer);
}
var ths = this;
this.setupTimer = setTimeout(function(){
ths.setupTimer = null;
try{
ths.main();
} catch(e) {
this.logger('error', 'debug', `[AardGl::scheduleInitRestart] <@${this.arid}> Failed to start main(). Error:`,e);
}
ths = null;
},
timeout
);
}
getTimeout(baseTimeout, startTime){
var execTime = (performance.now() - startTime);
return baseTimeout;
}
async nextFrame() {
return new Promise(resolve => window.requestAnimationFrame(resolve));
}
getDefaultAr() {
return this.video.videoWidth / this.video.videoHeight;
}
resetBlackLevel(){
this.blackLevel = this.settings.active.aard.blackbar.blackLevel;
}
clearImageData(id) {
if (ArrayBuffer.transfer) {
ArrayBuffer.transfer(id, 0);
}
id = undefined;
}
//#endregion
//#region canvas management
attachCanvas(canvas){
if(this.attachedCanvas)
this.attachedCanvas.remove();
// todo: place canvas on top of the video instead of random location
canvas.style.position = "absolute";
canvas.style.left = "200px";
canvas.style.top = "1200px";
canvas.style.zIndex = 10000;
document.getElementsByTagName("body")[0]
.appendChild(canvas);
}
canvasReadyForDrawWindow(){
this.logger.log('info', 'debug', `%c[AardGl::canvasReadyForDrawWindow] <@${this.arid}> canvas is ${this.canvas.height === window.innerHeight ? '' : 'NOT '}ready for drawWindow(). Canvas height: ${this.canvas.height}px; window inner height: ${window.innerHeight}px.`)
return this.canvas.height == window.innerHeight
}
//#endregion
//#region aard control
start() {
this.logger.log('info', 'debug', `"%c[AardGl::start] <@${this.arid}> Starting automatic aspect ratio detection`, _ard_console_start);
if (this.conf.resizer.lastAr.type === AspectRatio.Automatic) {
// ensure first autodetection will run in any case
this.conf.resizer.setLastAr({type: AspectRatio.Automatic, ratio: this.getDefaultAr()});
}
// launch main() if it's currently not running:
this.main();
// automatic detection starts halted. If halted=false when main first starts, extension won't run
// this._paused is undefined the first time we run this function, which is effectively the same thing
// as false. Still, we'll explicitly fix this here.
this._paused = false;
this._halted = false;
this._paused = false;
}
stop(){
this.logger.log('info', 'debug', `"%c[AardGl::stop] <@${this.arid}> Stopping automatic aspect ratio detection`, _ard_console_stop);
this._halted = true;
// this.conf.resizer.setArLastAr();
}
pause() {
// pause only if we were running before. Don't pause if we aren't running
// (we are running when _halted is neither true nor undefined)
if (this._halted === false) {
this._paused = true;
}
}
unpause() {
// pause only if we were running before. Don't pause if we aren't running
// (we are running when _halted is neither true nor undefined)
if (this._paused && this._halted === false) {
this._paused = true;
}
}
setManualTick(manualTick) {
this._manualTicks = manualTick;
}
tick() {
this._nextTick = true;
}
//#endregion
//#region WebGL helpers
glSetRectangle(glContext, width, height) {
glContext.bufferData(glContext.ARRAY_BUFFER, new Float32Array([
0, 0,
width, 0,
0, height,
0, height,
width, 0,
width, height
]), glContext.STATIC_DRAW);
}
/**
* Creates shader
* @param {*} glContext — gl context
* @param {*} shaderSource — shader code (as returned by a shader generator, for example)
* @param {*} shaderType — shader type (gl[context].FRAGMENT_SHADER or gl[context].VERTEX_SHADER)
*/
compileShader(glContext, shaderSource, shaderType) {
const shader = glContext.createShader(shaderType);
// load source and compile shader
glContext.shaderSource(shader, shaderSource);
glContext.compileShader(shader);
// check if shader was compiled successfully
if (! glContext.getShaderParameter(shader, this.gl.COMPILE_STATUS)) {
glContext.deleteShader(shader);
this.logger.log('error', ['init', 'debug', 'arDetect'], `%c[AardGl::setupShader] <@${this.arid}> Failed to setup shader.`, _ard_console_stop);
return null;
}
return shader;
}
/**
* Creates gl program
* @param {*} glContext — gl context
* @param {*} shaders — shaders (previously compiled with setupShader())
*/
compileProgram(glContext, shaders) {
console.log(glContext, shaders);
const program = glContext.createProgram();
for (const shader of shaders) {
glContext.attachShader(program, shader);
}
glContext.linkProgram(program);
if (! glContext.getProgramParameter(program, glContext.LINK_STATUS)) {
glContext.deleteShader(shader);
this.logger.log('error', ['init', 'debug', 'arDetect'], `%c[AardGl::setupProgram] <@${this.arid}> Failed to setup program.`, _ard_console_stop);
return null;
}
return program;
}
//#endregion
/*
* --------------------
* SETUP AND CLEANUP
* --------------------
*/
//#region init and destroy
init(){
this.logger.log('info', 'init', `[AardGl::init] <@${this.arid}> Initializing autodetection.`);
try {
if (this.settings.canStartAutoAr()) {
this.setup();
} else {
throw "Settings prevent autoar from starting"
}
} catch (e) {
this.logger.log('error', 'init', `%c[AardGl::init] <@${this.arid}> Initialization failed.`, _ard_console_stop, e);
}
}
destroy(){
this.logger.log('info', 'init', `%c[AardGl::destroy] <@${this.arid}> Destroying aard.`, _ard_console_stop, e);
// this.debugCanvas.destroy();
this.stop();
}
//#endregion
setup(cwidth, cheight){
this.logger.log('info', 'init', `[AardGl::setup] <@${this.arid}> Starting autodetection setup.`);
//
// [-1] check for zero-width and zero-height videos. If we detect this, we kick the proverbial
// can some distance down the road. This problem will prolly fix itself soon. We'll also
// not do any other setup until this issue is fixed
//
if(this.video.videoWidth === 0 || this.video.videoHeight === 0 ){
this.logger.log('warn', 'debug', `[AardGl::setup] <@${this.arid}> This video has zero width or zero height. Dimensions: ${this.video.videoWidth} × ${this.video.videoHeight}`);
this.scheduleInitRestart();
return;
}
//
// [0] initiate "dependencies" first
//
// This is space for EdgeDetector and GuardLine init
//
// [1] initiate canvases
//
if (!cwidth) {
cwidth = this.settings.active.aard.Gl.canvasDimensions.sampleCanvas.width;
cheight = this.settings.active.aard.Gl.canvasDimensions.sampleCanvas.height;
}
if (this.canvas) {
this.canvas.remove();
}
if (this.blackframeCanvas) {
this.blackframeCanvas.remove();
}
// things to note: we'll be keeping canvas in memory only.
this.canvas = document.createElement("canvas");
this.canvas.width = cwidth;
this.canvas.height = cheight;
this.blackframeCanvas = document.createElement("canvas");
this.blackframeCanvas.width = this.settings.active.aard.canvasDimensions.blackframeCanvas.width;
this.blackframeCanvas.height = this.settings.active.aard.canvasDimensions.blackframeCanvas.height;
// this.context = this.canvas.getContext("2d");
this.pixelBuffer = new Uint8Array(cwidth * cheight * 4);
//
// [2] SETUP WEBGL STUFF —————————————————————————————————————————————————————————————————————————————————
//#region webgl setup
this.gl = this.canvas.getContext("webgl");
// load shaders and stuff. PixelSize for horizontalAdder should be 1/sample canvas width
const vertexShaderSrc = getBasicVertexShader();
const horizontalAdderShaderSrc = generateHorizontalAdder(10, 1 / cwidth); // todo: unhardcode 10 as radius
// compile shaders
const vertexShader = this.compileShader(this.gl, vertexShaderSrc, this.gl.VERTEX_SHADER);
const horizontalAdderShader = this.compileShader(this.gl, horizontalAdderShaderSrc, this.gl.FRAGMENT_SHADER);
// link shaders to program
const glProgram = this.compileProgram(this.gl, [vertexShader, horizontalAdderShader]);
// look up where the vertex data needs to go
// const positionLocation = this.gl.getAttributeLocation(glProgram, 'a_position');
// const textureCoordsLocation = this.gl.getAttributeLocation(glProgram, 'a_textureCoords');
// create buffers and bind them
const positionBuffer = this.gl.createBuffer();
const textureCoordsBuffer = this.gl.createBuffer();
this.gl.bindBuffer(this.gl, positionBuffer);
this.gl.bindBuffer(this.gl.ARRAY_BUFFER, textureCoordsBuffer);
// create a texture
this.texture = this.gl.createTexture();
this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture);
// set some parameters
// btw we don't need to set gl.TEXTURE_WRAP_[S|T], because it's set to repeat by default — which is what we want
this.gl.texParameteri(this.gl, this.gl.TEXTURE_MAG_FILTER, this.gl.NEAREST);
this.gl.texParameteri(this.gl, this.gl.TEXTURE_MIN_FILTER, this.gl.NEAREST);
// we need a rectangle. This is output data, not texture. This means that the size of the rectangle should be
// [sample count] x height of the sample, as shader can sample frame at a different resolution than what gets
// rendered here. We don't need all horizontal pixels on our output. We do need all vertical pixels, though)
this.glSetRectangle(this.gl, this.settings.active.aard.sampleCols, cheight);
// do setup once
// tho we could do it for every frame
this.canvasScaleFactor = cheight / this.video.videoHeight;
//#endregion
//
// [3] detect if we're in the fallback mode and reset guardline
//
if (this.fallbackMode) {
this.logger.log('warn', 'debug', `[AardGl::setup] <@${this.arid}> WARNING: CANVAS RESET DETECTED/we're in fallback mode - recalculating guardLine`, "background: #000; color: #ff2");
// blackbar, imagebar
this.guardLine.reset();
}
//
// [4] see if browser supports "fallback mode" by drawing a small portion of our window
//
try {
this.blackframeContext.drawWindow(window,0, 0, this.blackframeCanvas.width, this.blackframeCanvas.height, "rgba(0,0,128,1)");
this.canDoFallbackMode = true;
} catch (e) {
this.canDoFallbackMode = false;
}
//
// [5] do other things setup needs to do
//
this.detectionTimeoutEventCount = 0;
this.resetBlackLevel();
// if we're restarting AardGl, we need to do this in order to force-recalculate aspect ratio
this.conf.resizer.setLastAr({type: AspectRatio.Automatic, ratio: this.getDefaultAr()});
this.canvasImageDataRowLength = cwidth << 2;
this.noLetterboxCanvasReset = false;
if (this.settings.canStartAutoAr() ) {
this.start();
}
if(Debug.debugCanvas.enabled){
// this.debugCanvas.init({width: cwidth, height: cheight});
// DebugCanvas.draw("test marker","test","rect", {x:5, y:5}, {width: 5, height: 5});
}
this.conf.arSetupComplete = true;
console.log("DRAWING BUFFER SIZE:", this.gl.drawingBufferWidth, '×', this.gl.drawingBufferHeight);
}
drawFrame() {
const level = 0;
const internalFormat = this.gl.RGBA;
const sourceFormat = this.gl.RGBA;
const sourceType = this.gl.UNSIGNED_BYTE;
this.gl.bindTexture(this.gl.TEXTURE_2D, this.texture);
// if (this.resizeInput) {
// TODO: check if 'width' and 'height' mean the input gets resized
// this.gl.texImage2D(gl.TEXTURE_2D, level, internalformat, width, height, border, format, type, pixels)
// } else {
this.gl.texImage2D(gl.TEXTURE_2D, level, internalformat, sourceFormat, sourceType, this.video);
// }
// get the pixels back out:
this.gl.readPixels(0, 0, width, height, format, type, pixels)
}
async main() {
if (this._paused) {
// unpause if paused
this._paused = false;
return; // main loop still keeps executing. Return is needed to avoid a million instances of autodetection
}
if (!this._halted) {
// we are already running, don't run twice
// this would have handled the 'paused' from before, actually.
return;
}
let exitedRetries = 10;
while (!this._exited && exitedRetries --> 0) {
this.logger.log('warn', 'debug', `[AardGl::main] <@${this.arid}> We are trying to start another instance of autodetection on current video, but the previous instance hasn't exited yet. Waiting for old instance to exit ...`);
await sleep(this.settings.active.aard.timers.tickrate);
}
if (!this._exited) {
this.logger.log('error', 'debug', `[AardGl::main] <@${this.arid}> Previous instance didn't exit in time. Not starting a new one.`);
return;
}
this.logger.log('info', 'debug', `%c[AardGl::main] <@${this.arid}> Previous instance didn't exit in time. Not starting a new one.`);
// we need to unhalt:
this._halted = false;
this._exited = false;
// set initial timestamps so frame check will trigger the first time we run the loop
let lastFrameCheckStartTime = Date.now() - (this.settings.active.aard.Gl.timers.playing << 1);
const frameCheckTimes = new Array(10).fill(-1);
let frameCheckBufferIndex = 0;
let fcstart, fctime;
while (this && !this._halted) {
// NOTE: we separated tickrate and inter-check timeouts so that when video switches
// state from 'paused' to 'playing', we don't need to wait for the rest of the longer
// paused state timeout to finish.
if ( (!this._manualTicks && this.canTriggerFrameCheck(lastFrameCheckStartTime)) || this._nextTick) {
this._nextTick = false;
lastFrameCheckStartTime = Date.now();
fcstart = performance.now();
try {
this.frameCheck();
} catch (e) {
this.logger.log('error', 'debug', `%c[AardGl::main] <@${this.arid}> Frame check failed:`, "color: #000, background: #f00", e);
}
if (Debug.performanceMetrics) {
fctime = performance.now() - fcstart;
frameCheckTimes[frameCheckBufferIndex % frameCheckTimes.length] = fctime;
this.conf.pageInfo.sendPerformanceUpdate({frameCheckTimes: frameCheckTimes, lastFrameCheckTime: fctime});
++frameCheckBufferIndex;
}
}
await this.nextFrame();
}
this.logger.log('info', 'debug', `%c[AardGl::main] <@${this.arid}> Main autodetection loop exited. Halted? ${this._halted}`, _ard_console_stop);
this._exited = true;
}
frameCheck(){
if(! this.video){
this.logger.log('error', 'debug', `%c[AardGl::frameCheck] <@${this.arid}> Video went missing. Destroying current instance of videoData.`);
this.conf.destroy();
return;
}
if (!this.blackframeContext) {
this.init();
}
var startTime = performance.now();
//
// [0] try drawing image to canvas
//
let imageData;
try {
this.drawFrame();
this.fallbackMode = false;
} catch (e) {
this.logger.log('error', 'arDetect', `%c[AardGl::frameCheck] <@${this.arid}> %c[AardGl::frameCheck] can't draw image on canvas. ${this.canDoFallbackMode ? 'Trying canvas.drawWindow instead' : 'Doing nothing as browser doesn\'t support fallback mode.'}`, "color:#000; backgroud:#f51;", e);
}
// [1]
this.clearImageData(imageData);
}
/**
* -------------------------
* DATA PROCESSING HELPERS
* -------------------------
*/
//#region result processing
calculateArFromEdges(edges) {
// if we don't specify these things, they'll have some default values.
if(edges.top === undefined){
edges.top = 0;
edges.bottom = 0;
edges.left = 0; // RESERVED FOR FUTURE — CURRENTLY UNUSED
edges.right = 0; // THIS FUNCTION CAN PRESENTLY ONLY HANDLE LETTERBOX
}
let letterbox = edges.top + edges.bottom;
if (! this.fallbackMode) {
// Since video is stretched to fit the canvas, we need to take that into account when calculating target
// aspect ratio and correct our calculations to account for that
const fileAr = this.video.videoWidth / this.video.videoHeight;
const canvasAr = this.canvas.width / this.canvas.height;
let widthCorrected;
if (edges.top && edges.bottom) {
// in case of letterbox, we take canvas height as canon and assume width got stretched or squished
if (fileAr != canvasAr) {
widthCorrected = this.canvas.height * fileAr;
} else {
widthCorrected = this.canvas.width;
}
return widthCorrected / (this.canvas.height - letterbox);
}
} else {
// fallback mode behaves a wee bit differently
let zoomFactor = 1;
// there's stuff missing from the canvas. We need to assume canvas' actual height is bigger by a factor x, where
// x = [video.zoomedHeight] / [video.unzoomedHeight]
//
// letterbox also needs to be corrected:
// letterbox += [video.zoomedHeight] - [video.unzoomedHeight]
var vbr = this.video.getBoundingClientRect();
zoomFactor = vbr.height / this.video.clientHeight;
letterbox += vbr.height - this.video.clientHeight;
var trueHeight = this.canvas.height * zoomFactor - letterbox;
if(edges.top > 1 && edges.top <= this.settings.active.aard.fallbackMode.noTriggerZonePx ){
this.logger.log('info', 'arDetect', `%c[AardGl::calculateArFromEdges] <@${this.arid}> Edge is in the no-trigger zone. Aspect ratio change is not triggered.`)
return;
}
// varnostno območje, ki naj ostane črno (da lahko v fallback načinu odkrijemo ožanje razmerja stranic).
// x2, ker je safetyBorderPx definiran za eno stran.
// safety border so we can detect aspect ratio narrowing (21:9 -> 16:9).
// x2 because safetyBorderPx is for one side.
trueHeight += (this.settings.active.aard.fallbackMode.safetyBorderPx << 1);
return this.canvas.width * zoomFactor / trueHeight;
}
}
processAr(trueAr){
this.detectedAr = trueAr;
// poglejmo, če se je razmerje stranic spremenilo
// check if aspect ratio is changed:
var lastAr = this.conf.resizer.getLastAr();
if (lastAr.type === AspectRatio.Automatic && lastAr.ratio !== null){
// spremembo lahko zavrnemo samo, če uporabljamo avtomatski način delovanja in če smo razmerje stranic
// že nastavili.
//
// we can only deny aspect ratio changes if we use automatic mode and if aspect ratio was set from here.
var arDiff = trueAr - lastAr.ar;
if (arDiff < 0)
arDiff = -arDiff;
var arDiff_percent = arDiff / trueAr;
// ali je sprememba v mejah dovoljenega? Če da -> fertik
// is ar variance within acceptable levels? If yes -> we done
this.logger.log('info', 'arDetect', `%c[AardGl::processAr] <@${this.arid}> New aspect ratio varies from the old one by this much:\n`,"color: #aaf","old Ar", lastAr.ar, "current ar", trueAr, "arDiff (absolute):",arDiff,"ar diff (relative to new ar)", arDiff_percent);
if (arDiff < trueAr * this.settings.active.aard.allowedArVariance){
this.logger.log('info', 'arDetect', `%c[AardGl::processAr] <@${this.arid}> Aspect ratio change denied — diff %: ${arDiff_percent}`, "background: #740; color: #fa2");
return;
}
this.logger.log('info', 'arDetect', `%c[AardGl::processAr] <@${this.arid}> aspect ratio change accepted — diff %: ${arDiff_percent}`, "background: #153; color: #4f9");
}
this.logger.log('info', 'debug', `%c[AardGl::processAr] <@${this.arid}> Triggering aspect ratio change. New aspect ratio: ${trueAr}`, _ard_console_change);
this.conf.resizer.updateAr({type: AspectRatio.Automatic, ratio: trueAr}, {type: AspectRatio.Automatic, ratio: trueAr});
}
//#endregion
//#region data processing / frameCheck helpers
//#endregion
}
var _ard_console_stop = "background: #000; color: #f41";
var _ard_console_start = "background: #000; color: #00c399";
var _ard_console_change = "background: #000; color: #ff8";
export default AardGl;