From 069f81d2ed1e9d55d20dcc9e23b5793099526748 Mon Sep 17 00:00:00 2001 From: Tamius Han Date: Wed, 15 Jan 2025 23:24:04 +0100 Subject: [PATCH] Prepare things for iframe transparency checkign --- .vscode/settings.json | 1 + package-lock.json | 43 +++- package.json | 1 + src/common/interfaces/SettingsInterface.ts | 50 ++-- .../InPlayerUiAdvertisement.vue | 4 +- src/ext/UWServer.ts | 44 +++- src/ext/conf/ExtensionConf.ts | 3 + src/ext/lib/IframeTransparencyVerifier.ts | 217 ++++++++++++++++++ src/ext/lib/comms/ChromeTabInterfaces.ts | 31 +++ src/ext/lib/comms/CommsClient.ts | 23 ++ 10 files changed, 391 insertions(+), 26 deletions(-) create mode 100644 src/ext/lib/IframeTransparencyVerifier.ts create mode 100644 src/ext/lib/comms/ChromeTabInterfaces.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index a81936e..dc1fd54 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -16,6 +16,7 @@ "decycle", "dinked", "dinks", + "Discardable", "disneyplus", "endregion", "equalish", diff --git a/package-lock.json b/package-lock.json index 83682bc..aa2eb49 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3243,8 +3243,7 @@ "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "at-least-node": { "version": "1.0.0", @@ -3375,6 +3374,28 @@ "integrity": "sha512-3AungXC4I8kKsS9PuS4JH2nc+0bVY/mjgrephHTIi8fpEeGsTHBUJeosp0Wc1myYMElmD0B3Oc4XL/HVJ4PV2g==", "dev": true }, + "axios": { + "version": "1.7.9", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.9.tgz", + "integrity": "sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw==", + "requires": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + }, + "dependencies": { + "form-data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } + } + }, "babel-code-frame": { "version": "6.26.0", "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz", @@ -4849,7 +4870,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "requires": { "delayed-stream": "~1.0.0" } @@ -5957,8 +5977,7 @@ "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, "delegates": { "version": "1.0.0", @@ -6997,6 +7016,11 @@ "readable-stream": "^2.3.6" } }, + "follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==" + }, "for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -9375,14 +9399,12 @@ "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==" }, "mime-types": { "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, "requires": { "mime-db": "1.52.0" } @@ -12810,6 +12832,11 @@ "ipaddr.js": "1.9.1" } }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", diff --git a/package.json b/package.json index baf8768..503e96d 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@mdi/font": "^6.5.95", "@mdi/js": "^6.4.95", "@types/resize-observer-browser": "^0.1.6", + "axios": "^1.7.9", "concurrently": "^5.3.0", "fs-extra": "^7.0.1", "gl-matrix": "^3.4.3", diff --git a/src/common/interfaces/SettingsInterface.ts b/src/common/interfaces/SettingsInterface.ts index 7d21eef..dc5008d 100644 --- a/src/common/interfaces/SettingsInterface.ts +++ b/src/common/interfaces/SettingsInterface.ts @@ -164,20 +164,7 @@ interface SettingsInterface { arDetect: AardSettings, ui: { - inPlayer: { - enabled: boolean, - enabledFullscreenOnly: boolean, - popupAlignment: 'left' | 'right', - minEnabledWidth: number, // don't show UI if player is narrower than % of screen width - minEnabledHeight: number, // don't show UI if player is narrower than % of screen height - activation: 'trigger-zone' | 'player', // what needs to be hovered in order for UI to be visible - triggerZoneDimensions: { // how large the trigger zone is (relative to player size) - width: number - height: number, - offsetX: number, // fed to translateX(offsetX + '%'). Valid range [-100, 0] - offsetY: number // fed to translateY(offsetY + '%'). Valid range [-100, 100] - }, - } + inPlayer: InPlayerUISettingsInterface } restrictions?: RestrictionsSettings; @@ -305,13 +292,16 @@ interface SettingsInterface { // sites: { [x: string]: SiteSettingsInterface, - } + }, + + telemetry?: UwTelemetryInterface } export interface SiteSettingsInterface { enable: ExtensionEnvironmentSettingsInterface; enableAard: ExtensionEnvironmentSettingsInterface; enableKeyboard: ExtensionEnvironmentSettingsInterface; + ui?: InPlayerUISettingsInterface, type?: 'official' | 'community' | 'user-defined' | 'testing' | 'officially-disabled' | 'unknown' | 'modified'; defaultType: 'official' | 'community' | 'user-defined' | 'testing' | 'officially-disabled' | 'unknown' | 'modified'; @@ -340,6 +330,12 @@ export interface SiteSettingsInterface { // the following fields are for use with extension update script override?: boolean; // whether settings for this site will be overwritten by extension upgrade script + + workarounds?: { + disableColorSchemeAwareness?: boolean; + forceColorScheme?: 'normal' | 'light' | 'dark', + lastColorSchemeAwarenessCheck?: Date; + } } export interface PlayerAutoConfigInterface { @@ -372,4 +368,28 @@ export interface SiteDOMElementSettingsInterface { nodeCss?: {[x: string]: string}; } +export interface InPlayerUISettingsInterface { + enabled: boolean, + enabledFullscreenOnly: boolean, + popupAlignment: 'left' | 'right', + minEnabledWidth: number, // don't show UI if player is narrower than % of screen width + minEnabledHeight: number, // don't show UI if player is narrower than % of screen height + activation: 'trigger-zone' | 'player', // what needs to be hovered in order for UI to be visible + triggerZoneDimensions: { // how large the trigger zone is (relative to player size) + width: number + height: number, + offsetX: number, // fed to translateX(offsetX + '%'). Valid range [-100, 0] + offsetY: number // fed to translateY(offsetY + '%'). Valid range [-100, 100] + }, +} + +export interface UwTelemetryInterface { + iframeTransparency?: { + [siteName: string]: { + reportedWithColorSchemeAllowed?: boolean; + reportedWithColorSchemeDisabled?: boolean; + } + } +} + export default SettingsInterface; diff --git a/src/csui/src/PlayerUiPanels/InPlayerUiAdvertisement.vue b/src/csui/src/PlayerUiPanels/InPlayerUiAdvertisement.vue index 4913176..9964393 100644 --- a/src/csui/src/PlayerUiPanels/InPlayerUiAdvertisement.vue +++ b/src/csui/src/PlayerUiPanels/InPlayerUiAdvertisement.vue @@ -62,7 +62,9 @@ export default { 'uw-page-stats': { function: (data) => { console.log('got page statss:', data); - this.pageData = data; + this.pageData = JSON.parse(JSON.stringify(data)); + + this.$nextTick( () => this.$forceUpdate()); } } }, diff --git a/src/ext/UWServer.ts b/src/ext/UWServer.ts index ab138e8..59c4c43 100644 --- a/src/ext/UWServer.ts +++ b/src/ext/UWServer.ts @@ -5,6 +5,9 @@ import Settings from './lib/Settings'; import Logger, { baseLoggingOptions } from './lib/Logger'; import { sleep } from '../common/js/utils'; import EventBus, { EventBusCommand } from './lib/EventBus'; +import { IframeTransparencyVerifier, IframeVerificationPlayerData } from './lib/IframeTransparencyVerifier'; +import { MessageSender } from './lib/comms/ChromeTabInterfaces'; +import axios from 'axios'; export default class UWServer { settings: Settings; @@ -18,6 +21,8 @@ export default class UWServer { videoTabs: any = {}; currentTabId: number = 0; + IframeTransparencyVerifier: IframeTransparencyVerifier = new IframeTransparencyVerifier() + selectedSubitem: any = { 'siteSettings': undefined, 'videoSettings': undefined, @@ -44,6 +49,11 @@ export default class UWServer { }, 'get-current-site': { function: (message, context) => this.getCurrentSite() + }, + 'verify-iframe-transparency': { + function: (message, context) => { + this.verifyUiTransparency(message.playerData, context.comms.sender, message.telemetryData); + } } }; @@ -101,7 +111,7 @@ export default class UWServer { }); } - async injectCss(css, sender) { + async injectCss(css, sender: MessageSender) { if (!css) { return; } @@ -133,7 +143,7 @@ export default class UWServer { this.logger.log('error','debug', '[UwServer::injectCss] Error while injecting css:', {error: e, css, sender}); } } - async removeCss(css, sender) { + async removeCss(css, sender: MessageSender) { try { if (BrowserDetect.firefox) { chrome.scripting.removeCSS({ @@ -367,4 +377,34 @@ export default class UWServer { return hostname; } + + async verifyUiTransparency(verificationData: IframeVerificationPlayerData, sender: MessageSender, telemetryData?: any) { + const hasTransparency = this.IframeTransparencyVerifier.verifyUiTransparency( + sender.tab.windowId, + { + player: verificationData, + tab: { + width: sender.tab.width, + height: sender.tab.height, + } + } + ); + + this.eventBus.send( + 'iframe-transparency-results', + { + comms: { + forwardTo: 'active' + } + } + ); + + axios.post( + 'https://uw-telemetry.tamius.net/iframe-transparency', + { + ...telemetryData, + transparencyCheckResult: hasTransparency, + } + ); + } } diff --git a/src/ext/conf/ExtensionConf.ts b/src/ext/conf/ExtensionConf.ts index afa34fb..d0c4c8e 100644 --- a/src/ext/conf/ExtensionConf.ts +++ b/src/ext/conf/ExtensionConf.ts @@ -1842,6 +1842,9 @@ const ExtensionConf: SettingsInterface = { } } }, + }, + + telemetry: { } } diff --git a/src/ext/lib/IframeTransparencyVerifier.ts b/src/ext/lib/IframeTransparencyVerifier.ts new file mode 100644 index 0000000..f1add8b --- /dev/null +++ b/src/ext/lib/IframeTransparencyVerifier.ts @@ -0,0 +1,217 @@ +export enum TransparencyVerificationResult { + Ok = 0, + Fail = 1, + NoVisibleElements = 2, + Error = 3 +} + +export interface IframeVerificationData { + tab: { + width: number, + height: number, + }, + player: IframeVerificationPlayerData +} + +export interface IframeVerificationPlayerData { + x: number, + y: number, + width: number, + height: number, +} + +interface IframeCheckItem { + position: 'top' | 'center' | 'bottom' | 'left' | 'right'; + offset: number; +} + +interface IframeCheckPosition { + positionX: 'top' | 'center' | 'bottom'; + positionY: 'left' | 'center' | 'right'; + offsetX: number; + offsetY: number; +} + +export class IframeTransparencyVerifier { + + async verifyUiTransparency(windowId: number, tabDimensions: IframeVerificationData): Promise { + const {visibleX, visibleY} = this.getVisibleMarkers(tabDimensions); + if (!visibleX.length || visibleY.length) { + return TransparencyVerificationResult.NoVisibleElements; + } + + const checkPositions = this.processMarkers(visibleX, visibleY); + + const dataUrl = await chrome.tabs.captureVisibleTab( + windowId, + { + format: "png" + } + ); + + try { + const canvas = new OffscreenCanvas(tabDimensions.tab.width, tabDimensions.tab.height); + const ctx = canvas.getContext('2d')!; + + const image = new Image(); + const imageLoadPromise = new Promise(r => image.onload = r); + image.src = dataUrl; + await imageLoadPromise; + (ctx as any).drawImage(image, 0, 0); + + const imageData = (ctx as any).getImageData(0, 0, tabDimensions.tab.width, tabDimensions.tab.height).data; + + if (this.detectMarkers(checkPositions, tabDimensions.tab.width, imageData)) { + return TransparencyVerificationResult.Ok; + } else { + return TransparencyVerificationResult.Fail; + } + } catch (e) { + return TransparencyVerificationResult.Error; + } + } + + private getVisibleMarkers({tab, player}: IframeVerificationData) { + const visibleX: IframeCheckItem[] = []; + const visibleY: IframeCheckItem[] = []; + + // Determine which markers should be visible. + + // Visibility: TOP ROW + if (player.y >= 0 && player.y < tab.height) { + visibleY.push({ + position: 'top', + offset: player.y, + }); + } + + // Visibility: CENTER + const yHalfPosition = Math.floor(player.y + player.height / 2); + if (player.y + yHalfPosition - 2 > 0 || player.y + yHalfPosition + 2 < tab.height) { + visibleY.push({ + position: 'center', + offset: player.y + yHalfPosition, + }); + } + + // Visibility: BOTTOM ROW + if (player.y + player.height - 1 > 0 || player.y + player.height + 1 < tab.height) { + visibleY.push({ + position: 'bottom', + offset: player.y + player.height - 1, + }); + } + + // Visibility: LEFT SIDE + if (player.x >= 0 && player.x < tab.width) { + visibleX.push({ + position: 'left', + offset: player.x, + }); + } + + // Visibility: X CENTER + const xHalfPosition = Math.floor(player.x + player.width / 2); + if (player.x + xHalfPosition - 2 > 0 || player.x + xHalfPosition + 2 < tab.width) { + visibleY.push({ + position: 'center', + offset: player.x + xHalfPosition, + }); + } + + // Visibility: RIGHT SIDE + if (player.x + player.width - 1 > 0 || player.x + player.width + 1 < tab.width) { + visibleX.push({ + position: 'right', + offset: player.x + player.width - 1, + }); + } + + return { + visibleX, + visibleY, + }; + + } + + private processMarkers(candidatesX: IframeCheckItem[], candidatesY: IframeCheckItem[]): IframeCheckPosition[] { + if (!candidatesX.length || !candidatesY.length) { + return [] as IframeCheckPosition[]; + } + + const checkPositions: IframeCheckPosition[] = []; + + for (const row of candidatesY) { + for (const col of candidatesX) { + // 'center center' is not valid position. + if (row.position !== col.position) { + checkPositions.push({ + positionX: row.position as 'top' | 'center' | 'bottom', + positionY: col.position as 'left' | 'center' | 'right', + offsetX: col.offset, + offsetY: row.offset, + }); + } + } + } + + return checkPositions; + } + + private detectMarkers(checkPositions: IframeCheckPosition[], rowLength: number, imageData: Uint8ClampedArray): boolean { + for (const position of checkPositions) { + if (position.positionY === 'center') { + if (this.detectColumnMarker(position.offsetX, position.offsetY, rowLength, imageData)) { + return true; + } + } else { + if (this.detectRowMarker(position.offsetX, position.offsetY, rowLength, imageData)) { + return true; + } + } + } + + return false; + } + + // Checks if our magic sequence is present in a row configuration + private detectRowMarker(x, y, rowLength, imageData): boolean { + const start = (y * rowLength + x) * 4; + + return imageData[start] === 0 + && imageData[start + 1] === 1 + && imageData[start + 2] === 2 + && imageData[start + 4] === 3 + && imageData[start + 5] === 4 + && imageData[start + 6] === 5 + && imageData[start + 8] === 5 + && imageData[start + 9] === 4 + && imageData[start + 10] === 3 + && imageData[start + 12] === 2 + && imageData[start + 13] === 1 + && imageData[start + 15] === 0; + } + + // Checks if our magic sequence is present in a column configuration + private detectColumnMarker(x, y, rowLength, imageData) { + const rowOffset = rowLength * 4; + + const r1 = (y * rowLength + x) * 4; + const r2 = r1 + rowOffset; + const r3 = r2 + rowOffset; + const r4 = r3 + rowOffset; + + return imageData[r1] === 0 + && imageData[r1 + 1] === 1 + && imageData[r1 + 2] === 2 + && imageData[r2 ] === 3 + && imageData[r2 + 1] === 4 + && imageData[r2 + 2] === 5 + && imageData[r3 ] === 5 + && imageData[r3 + 1] === 4 + && imageData[r3 + 2] === 3 + && imageData[r4 ] === 2 + && imageData[r4 + 1] === 1 + && imageData[r4 + 2] === 0; + } +} diff --git a/src/ext/lib/comms/ChromeTabInterfaces.ts b/src/ext/lib/comms/ChromeTabInterfaces.ts new file mode 100644 index 0000000..9c3dc12 --- /dev/null +++ b/src/ext/lib/comms/ChromeTabInterfaces.ts @@ -0,0 +1,31 @@ +export interface MessageSender { + documentId?: string, + frameId?: number, + id: string, + origin: string, + tab: Tab, + url: string, +} + +export interface Tab { + active: boolean, + audible?: boolean, + autoDiscardable: boolean, + discarded: boolean, + favIconUrl?: string, + frozen?: boolean, + groupId: number, + height: number, + highlighted: boolean, + id?: number, + incognito: boolean, + index: number, + lastAccessed: number, + openerTabId?: number, + pendingUrl?: string, + pinned: boolean, + sessionId: string, + url?: string; + width: number; + windowId?: number; +} diff --git a/src/ext/lib/comms/CommsClient.ts b/src/ext/lib/comms/CommsClient.ts index 141127f..fbc5127 100644 --- a/src/ext/lib/comms/CommsClient.ts +++ b/src/ext/lib/comms/CommsClient.ts @@ -117,6 +117,29 @@ class CommsClient { this.commsId = (Math.random() * 20).toFixed(0); + this.eventBus.subscribeMulti( + { + 'uw-get-page-stats': { + function: (config, routingData) => { + this.eventBus.send( + 'uw-page-stats', + { + pcsDark: window.matchMedia('(prefers-color-scheme: dark)').matches, + pcsLight: window.matchMedia('(prefers-color-scheme: light)').matches, + colorScheme: window.getComputedStyle( document.body ,null).getPropertyValue('color-scheme') + }, + { + comms: { + forwardTo: 'popup' + } + } + ); + } + }, + }, + this + ); + } catch (e) { console.error("CONSTRUCOTR FAILED:", e) }