tamius-han/ultrawidify
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switch to gl canvas

Browse Source
This commit is contained in:
Tamius Han 2024-10-10 00:45:28 +02:00
parent e56d32180c
commit de78f8b1d5
4 changed files with 203 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/ext/lib/aard/gl/GlCanvas.ts
View File

@ -82,6 +82,12 @@ export class GlCanvas {
private programInfo: GlCanvasProgramInfo;
private projectionMatrix: mat4;
get width() {
return this.canvas.width;
}
get height() {
return this.canvas.height;
}
constructor(options: GlCanvasOptions) {
this.canvas = document.createElement('canvas');
@ -117,6 +123,18 @@ export class GlCanvas {
return this.frameBuffer;
}
/**
* Cleans up after itself
*/
destroy() {
this.gl.deleteProgram(this.programInfo.program);
this.gl.deleteBuffer(this.buffers.position);
this.gl.deleteBuffer(this.buffers.normal);
this.gl.deleteBuffer(this.buffers.textureCoord);
this.gl.deleteBuffer(this.buffers.indices);
this.gl.deleteTexture(this.texture);
}
private initWebgl() {
// Initialize the GL context
this.gl.clearColor(0.0, 0.0, 0.0, 1.0);

80
src/ext/lib/ar-detect/ArDetector.ts
View File

@ -14,6 +14,7 @@ import Logger from '../Logger';
import VideoData from '../video-data/VideoData';
import Settings from '../Settings';
import EventBus from '../EventBus';
import { GlCanvas } from '../aard/gl/GlCanvas';
enum VideoPlaybackState {
Playing,
@ -82,11 +83,10 @@ class ArDetector {
_nextTick: boolean;
private animationFrameHandle: any;
private attachedCanvas: HTMLCanvasElement;
canvas: HTMLCanvasElement;
private context: CanvasRenderingContext2D;
canvasImageDataRowLength: number;
glCanvas: GlCanvas;
private timers = {
nextFrameCheckTime: Date.now()
}
@ -189,43 +189,24 @@ class ArDetector {
//
// [1] initiate canvases
//
if (!cwidth) {
cwidth = this.settings.active.arDetect.canvasDimensions.sampleCanvas.width;
cheight = this.settings.active.arDetect.canvasDimensions.sampleCanvas.height;
if (this.glCanvas) {
this.glCanvas.destroy();
}
if (this.canvas) {
this.canvas.remove();
}
// if (this.blackframeCanvas) {
// this.blackframeCanvas.remove();
// }
// things to note: we'll be keeping canvas in memory only.
this.canvas = document.createElement("canvas");
this.canvas.width = cwidth;
this.canvas.height = cheight;
// this.blackframeCanvas = document.createElement("canvas");
// this.blackframeCanvas.width = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.width;
// this.blackframeCanvas.height = this.settings.active.arDetect.canvasDimensions.blackframeCanvas.height;
this.context = this.canvas.getContext("2d");
this.glCanvas = new GlCanvas(this.settings.active.arDetect.canvasDimensions.sampleCanvas);
//
// [2] determine places we'll use to sample our main frame
//
let ncol = this.settings.active.arDetect.sampling.staticCols;
let nrow = this.settings.active.arDetect.sampling.staticRows;
let colSpacing = this.canvas.width / ncol;
let rowSpacing = (this.canvas.height << 2) / nrow;
let colSpacing = this.glCanvas.width / ncol;
let rowSpacing = (this.glCanvas.height << 2) / nrow;
this.sampleLines = [];
this.sampleCols = [];
for(let i = 0; i < ncol; i++){
for (let i = 0; i < ncol; i++){
if(i < ncol - 1)
this.sampleCols.push(Math.round(colSpacing * i));
else{
@ -233,7 +214,7 @@ class ArDetector {
}
}
for(let i = 0; i < nrow; i++){
for (let i = 0; i < nrow; i++){
if(i < ncol - 5)
this.sampleLines.push(Math.round(rowSpacing * i));
else{
@ -244,7 +225,6 @@ class ArDetector {
//
// [3] do other things setup needs to do
//
this.resetBlackLevel();
// if we're restarting ArDetect, we need to do this in order to force-recalculate aspect ratio
@ -253,11 +233,10 @@ class ArDetector {
this.canvasImageDataRowLength = cwidth << 2;
this.start();
if(Debug.debugCanvas.enabled){
// if(Debug.debugCanvas.enabled){
// this.debugCanvas.init({width: cwidth, height: cheight});
// DebugCanvas.draw("test marker","test","rect", {x:5, y:5}, {width: 5, height: 5});
}
// }
this.conf.arSetupComplete = true;
}
@ -391,20 +370,6 @@ class ArDetector {
);
}
private attachCanvas(canvas){
if(this.attachedCanvas)
this.attachedCanvas.remove();
// todo: place canvas on top of the video instead of random location
canvas.style.position = "absolute";
canvas.style.left = "200px";
canvas.style.top = "1200px";
canvas.style.zIndex = 10000;
document.getElementsByTagName("body")[0]
.appendChild(canvas);
}
/**
* Adds execution time sample for performance metrics
* @param performanceObject
@ -749,19 +714,19 @@ class ArDetector {
// aspect ratio and correct our calculations to account for that
const fileAr = this.video.videoWidth / this.video.videoHeight;
const canvasAr = this.canvas.width / this.canvas.height;
const canvasAr = this.glCanvas.width / this.glCanvas.height;
let widthCorrected;
if (edges.top && edges.bottom) {
// in case of letterbox, we take canvas height as canon and assume width got stretched or squished
if (fileAr != canvasAr) {
widthCorrected = this.canvas.height * fileAr;
widthCorrected = this.glCanvas.height * fileAr;
} else {
widthCorrected = this.canvas.width;
widthCorrected = this.glCanvas.width;
}
return widthCorrected / (this.canvas.height - letterbox);
return widthCorrected / (this.glCanvas.height - letterbox);
}
}
@ -822,22 +787,19 @@ class ArDetector {
return;
}
if (!this.context) {
if (!this.glCanvas) {
this.init();
}
let sampleCols = this.sampleCols.slice(0);
let startTime = performance.now();
await new Promise<void>(
const imageData = await new Promise<Uint8Array>(
resolve => {
this.context.drawImage(this.video, 0, 0, this.canvas.width, this.canvas.height);
resolve();
this.glCanvas.drawVideoFrame(this.video);
resolve(this.glCanvas.getImageData());
}
)
const imageData = this.context.getImageData(0, 0, this.canvas.width, this.canvas.height).data;
timerResults.imageDrawTime = performance.now() - startTime;
startTime = performance.now();
@ -1130,7 +1092,7 @@ class ArDetector {
// returns 'false' if we found a non-black edge pixel.
// If we detect anything darker than blackLevel, we modify blackLevel to the new lowest value
const rowOffset = this.canvas.width * (this.canvas.height - 1);
const rowOffset = this.glCanvas.width * (this.glCanvas.height - 1);
let currentMin = 255, currentMax = 0, colOffset_r, colOffset_g, colOffset_b, colOffset_rb, colOffset_gb, colOffset_bb, blthreshold = this.settings.active.arDetect.blackbar.threshold;
// detect black level. if currentMax comes above blackbar + blackbar threshold, we know we aren't letterboxed

22
src/ext/lib/ar-detect/GuardLine.ts
View File

@ -42,7 +42,7 @@ class GuardLine {
// to odstrani vse neveljavne nastavitve in vse možnosti, ki niso smiselne
// this removes any configs with invalid values or values that dont make sense
if (bbTop < 0 || bbBottom >= this.aard.canvas.height ){
if (bbTop < 0 || bbBottom >= this.aard.glCanvas.height ){
throw {error: "INVALID_SETTINGS_IN_GUARDLINE", bbTop, bbBottom}
}
@ -99,7 +99,7 @@ class GuardLine {
return { success: true };
}
let offset = (this.aard.canvas.width * this.settings.active.arDetect.guardLine.ignoreEdgeMargin) << 2;
let offset = (this.aard.glCanvas.width * this.settings.active.arDetect.guardLine.ignoreEdgeMargin) << 2;
let offenders = [];
let offenderCount = -1; // doing it this way means first offender has offenderCount==0. Ez index.
@ -117,8 +117,8 @@ class GuardLine {
// <<<=======| checking upper row |========>>>
rowStart = ((edge_upper * this.aard.canvas.width) << 2) + offset;
rowEnd = rowStart + ( this.aard.canvas.width << 2 ) - (offset * 2);
rowStart = ((edge_upper * this.aard.glCanvas.width) << 2) + offset;
rowEnd = rowStart + ( this.aard.glCanvas.width << 2 ) - (offset * 2);
if (Debug.debugCanvas.enabled && Debug.debugCanvas.guardLine) {
// offenderCount = this._gl_debugRowCheck(image, rowStart, rowEnd, offenders, offenderCount);
@ -127,8 +127,8 @@ class GuardLine {
}
// <<<=======| checking lower row |========>>>
rowStart = ((edge_lower * this.aard.canvas.width) << 2) + offset;
rowEnd = rowStart + ( this.aard.canvas.width << 2 ) - (offset * 2);
rowStart = ((edge_lower * this.aard.glCanvas.width) << 2) + offset;
rowEnd = rowStart + ( this.aard.glCanvas.width << 2 ) - (offset * 2);
if (Debug.debugCanvas.enabled && Debug.debugCanvas.guardLine) {
// offenderCount = this._gl_debugRowCheck(image, rowStart, rowEnd, offenders, offenderCount);
@ -158,7 +158,7 @@ class GuardLine {
if(!this.imageBar.top || !this.imageBar.bottom)
return { success: false };
let offset = (this.aard.canvas.width * this.settings.active.arDetect.guardLine.ignoreEdgeMargin) << 2;
let offset = (this.aard.glCanvas.width * this.settings.active.arDetect.guardLine.ignoreEdgeMargin) << 2;
// TODO: implement logo check.
@ -167,14 +167,14 @@ class GuardLine {
// how many non-black pixels we need to consider this check a success. We only need to detect enough pixels
// on one edge (one of the edges can be black as long as both aren't)
let successThreshold = (this.aard.canvas.width * this.settings.active.arDetect.guardLine.imageTestThreshold);
let successThreshold = (this.aard.glCanvas.width * this.settings.active.arDetect.guardLine.imageTestThreshold);
let rowStart, rowEnd;
// <<<=======| checking upper row |========>>>
rowStart = ((edge_upper * this.aard.canvas.width) << 2) + offset;
rowEnd = rowStart + ( this.aard.canvas.width << 2 ) - (offset * 2);
rowStart = ((edge_upper * this.aard.glCanvas.width) << 2) + offset;
rowEnd = rowStart + ( this.aard.glCanvas.width << 2 ) - (offset * 2);
let res = false;
@ -190,7 +190,7 @@ class GuardLine {
// <<<=======| checking lower row |========>>>
rowStart = ((edge_lower * this.aard.canvas.width) << 2) + offset;
rowStart = ((edge_lower * this.aard.glCanvas.width) << 2) + offset;
// rowEnd = rowStart + ( this.conf.canvas.width << 2 ) - (offset * 2);

42
src/ext/lib/ar-detect/edge-detect/EdgeDetect.ts
View File

@ -100,7 +100,7 @@ class EdgeDetect{
if(guardLineOut){
if(guardLineOut.imageFail && !guardLineOut.blackbarFail && this.conf.guardLine.blackbar.top) {
upper_top = this.conf.guardLine.blackbar.top;
upper_bottom = this.conf.canvas.height >> 1;
upper_bottom = this.conf.glCanvas.height >> 1;
lower_top = upper_bottom;
lower_bottom = this.conf.guardLine.blackbar.bottom;
} else if (! guardLineOut.imageFail && !guardLineOut.blackbarFail && this.conf.guardLine.blackbar.top) {
@ -110,20 +110,20 @@ class EdgeDetect{
// a logo could falsely trigger this case, so we need to add some extra margins past
// the point marked by guardLine.blackbar. Let's say 1/8 of canvas height on either side.
upper_top = 0;
upper_bottom = this.conf.guardLine.blackbar.top + (this.conf.canvas.height >> 3);
lower_top = this.conf.guardLine.blackbar.bottom - (this.conf.canvas.height >> 3);
lower_bottom = this.conf.canvas.height - 1;
upper_bottom = this.conf.guardLine.blackbar.top + (this.conf.glCanvas.height >> 3);
lower_top = this.conf.guardLine.blackbar.bottom - (this.conf.glCanvas.height >> 3);
lower_bottom = this.conf.glCanvas.height - 1;
} else {
upper_top = 0;
upper_bottom = (this.conf.canvas.height >> 1) /*- parseInt(this.conf.canvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
lower_top = (this.conf.canvas.height >> 1) /*+ parseInt(this.conf.canvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
lower_bottom = this.conf.canvas.height - 1;
upper_bottom = (this.conf.glCanvas.height >> 1) /*- parseInt(this.conf.glCanvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
lower_top = (this.conf.glCanvas.height >> 1) /*+ parseInt(this.conf.glCanvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
lower_bottom = this.conf.glCanvas.height - 1;
}
} else{
upper_top = 0;
upper_bottom = (this.conf.canvas.height >> 1) /*- parseInt(this.conf.canvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
lower_top = (this.conf.canvas.height >> 1) /*+ parseInt(this.conf.canvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
lower_bottom = this.conf.canvas.height - 1;
upper_bottom = (this.conf.glCanvas.height >> 1) /*- parseInt(this.conf.glCanvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
lower_top = (this.conf.glCanvas.height >> 1) /*+ parseInt(this.conf.glCanvas.height * this.settings.active.arDetect.edgeDetection.middleIgnoredArea);*/
lower_bottom = this.conf.glCanvas.height - 1;
}
this.logger.log('info', 'arDetect', '[EdgeDetect::findCandidates] searching for candidates on ranges', upper_top, '<->', upper_bottom, ';', lower_top, '<->', lower_bottom);
@ -157,7 +157,7 @@ class EdgeDetect{
// NOTE: this is very simple and will need to be reworked in case we ever
// go for quorum-based edge detection. (Probably not gonna happen)
const topPoint = {
row: this.conf.canvas.height,
row: this.conf.glCanvas.height,
gradient: false, // does current row have a gradient sample
noGradient: false, // does current row have 100% confirmed edge sample
}
@ -220,7 +220,7 @@ class EdgeDetect{
} else {
// if top gradient-only sample is closer to the edge than the bottom sample,
// validation also fails. Otherwise, we can assume success.
return (topPoint.row >= this.conf.canvas.height - bottomPoint.row);
return (topPoint.row >= this.conf.glCanvas.height - bottomPoint.row);
}
}
@ -229,7 +229,7 @@ class EdgeDetect{
// whether gradient-only result of bottom row is closer to the edge than than the top
// sample.
if (! bottomPoint.noGradient) {
return (topPoint.row < this.conf.canvas.height - bottomPoint.row);
return (topPoint.row < this.conf.glCanvas.height - bottomPoint.row);
}
return false;
@ -240,8 +240,8 @@ class EdgeDetect{
let edgeCandidatesBottom = {count: 0};
let detections;
let canvasWidth = this.conf.canvas.width;
let canvasHeight = this.conf.canvas.height;
let canvasWidth = this.conf.glCanvas.width;
let canvasHeight = this.conf.glCanvas.height;
let sampleStart, sampleEnd, loopEnd;
let sampleRow_black, sampleRow_color;
@ -352,7 +352,7 @@ class EdgeDetect{
edgePostprocess(edges){
let edgesTop = [];
let edgesBottom = [];
let alignMargin = this.conf.canvas.height * this.settings.active.arDetect.allowedMisaligned;
let alignMargin = this.conf.glCanvas.height * this.settings.active.arDetect.allowedMisaligned;
let missingEdge = edges.edgeCandidatesTopCount == 0 || edges.edgeCandidatesBottomCount == 0;
@ -377,7 +377,7 @@ class EdgeDetect{
for(let e in edges.edgeCandidatesBottom){
let edge = edges.edgeCandidatesBottom[e];
edgesBottom.push({
distance: this.conf.canvas.height - edge.offset,
distance: this.conf.glCanvas.height - edge.offset,
absolute: edge.offset,
count: edge.count
});
@ -542,11 +542,11 @@ class EdgeDetect{
blackbarThreshold = this.conf.blackLevel + this.settings.active.arDetect.blackbar.threshold;
let middleRowStart = (this.conf.canvas.height >> 1) * this.conf.canvas.width;
let middleRowEnd = middleRowStart + this.conf.canvas.width - 1;
let middleRowStart = (this.conf.glCanvas.height >> 1) * this.conf.glCanvas.width;
let middleRowEnd = middleRowStart + this.conf.glCanvas.width - 1;
let rowStart = middleRowStart << 2;
let midpoint = (middleRowStart + (this.conf.canvas.width >> 1)) << 2
let midpoint = (middleRowStart + (this.conf.glCanvas.width >> 1)) << 2
let rowEnd = middleRowEnd << 2;
let edge_left = -1, edge_right = -1;
@ -564,7 +564,7 @@ class EdgeDetect{
// check on the right
for(let i = rowEnd; i > midpoint; i-= 4){
if(image[i] > blackbarThreshold || image[i+1] > blackbarThreshold || image[i+2] > blackbarThreshold){
edge_right = this.conf.canvas.width - ((i - rowStart) >> 2);
edge_right = this.conf.glCanvas.width - ((i - rowStart) >> 2);
break;
}
}