适用于 AmbientLight 的三星电视 Tizen Grabber 应用程序

Question

我目前正在开展一个项目，该项目涉及实时捕获运行 Tizen 操作系统的三星电视的屏幕内容，并将颜色信息传输到连接的 Hyperion 服务器以用于环境照明。以下是我迄今为止所取得的成就和面临的挑战的总结： 我所取得的成就：

1. 捕获视频内容： 我成功实现了一个 Web 应用程序，该应用程序捕获应用程序中播放的视频内容，并以最小的延迟将其传输到 Hyperion 服务器。这可确保从视频内容中顺利、实时地捕捉色彩。
2. 实时传输： 捕获的视频帧经过高效处理并发送给 Hyperion 服务器，从而根据视频内容提供无缝的环境照明体验。

此外，如果有人想要我将视频屏幕截图发送到 Hyperion 所实现的代码，这里是代码（在 Tizen Studio 上运行） main.js:

// Main
document.addEventListener('DOMContentLoaded', function () {
    log('Document loaded and DOM fully parsed');
    document.getElementById('startCapture').addEventListener('click', startCapture);
    document.getElementById('remoteButton').addEventListener('click', startCapture);
    document.getElementById('testImageButton').addEventListener('click', sendTestImage);
    document.getElementById('sendColorButton').addEventListener('click', sendRandomColor);
    document.getElementById('startVideoCaptureButton').addEventListener('click', startVideoCapture);
    document.getElementById('stopVideoCaptureButton').addEventListener('click', stopVideoCapture);

    document.addEventListener('keydown', function(event) {
        switch (event.key) {
            case 'Enter':
                log('Remote control button pressed');
                document.activeElement.click();
                break;
            case 'ArrowUp':
                focusPreviousElement();
                break;
            case 'ArrowDown':
                focusNextElement();
                break;
        }
    });

    initializeWebSocket();
});

function focusPreviousElement() {
    let focusable = document.querySelectorAll('[tabindex]');
    let index = Array.prototype.indexOf.call(focusable, document.activeElement);
    if (index > 0) {
        focusable[index - 1].focus();
    }
}

function focusNextElement() {
    let focusable = document.querySelectorAll('[tabindex]');
    let index = Array.prototype.indexOf.call(focusable, document.activeElement);
    if (index < focusable.length - 1) {
        focusable[index + 1].focus();
    }
}

function startCapture() {
    log('Start Capture button clicked');
    captureScreen();
}

function captureScreen() {
    log('Capturing screen');
    const captureArea = document.getElementById('captureArea');
    const canvas = document.createElement('canvas');
    const context = canvas.getContext('2d');

    const fixedWidth = 120;
    const fixedHeight = 120;

    canvas.width = fixedWidth;
    canvas.height = fixedHeight;

    try {
        // Colores alternativos para asegurar que la imagen no sea negra
        const colors = ['red', 'green', 'blue', 'yellow', 'purple', 'orange'];
        const randomColor = colors[Math.floor(Math.random() * colors.length)];

        context.fillStyle = randomColor;
        context.fillRect(0, 0, canvas.width, canvas.height);

        const text = captureArea.innerText || captureArea.textContent;
        context.font = '30px Arial';
        context.fillStyle = 'white';
        context.textAlign = 'center';
        context.textBaseline = 'middle';
        context.fillText(text, canvas.width / 2, canvas.height / 2);

        const imageData = canvas.toDataURL('image/jpeg', 0.03).split(',')[1]; // Lower quality to 0.03

        log('Screenshot captured, Base64 length: ' + imageData.length);

        sendToHyperion(imageData);
    } catch (error) {
        log('Error capturing screen: ' + error);
    }
}

function sendTestImage() {
    log('Sending test image to Hyperion');
    const testImageUrl = 'https://eligeeducar.cl/content/uploads/2019/10/Imagen-que-dice-rojo-sobre-un-fondo-azul-y-azul-sobre-un-fondo-rojo--1920x550.jpg';

    fetch(testImageUrl)
        .then(response => response.blob())
        .then(blob => compressImage(blob))
        .then(imageData => sendToHyperion(imageData))
        .catch(error => {
            log('Error fetching test image: ' + error);
        });
}

function compressImage(blob) {
    return new Promise((resolve, reject) => {
        const reader = new FileReader();
        reader.onload = function () {
            const img = new Image();
            img.onload = function () {
                const canvas = document.createElement('canvas');
                const ctx = canvas.getContext('2d');
                const maxWidth = 120;
                const maxHeight = 120;
                let width = img.width;
                let height = img.height;

                if (width > height) {
                    if (width > maxWidth) {
                        height *= maxWidth / width;
                        width = maxWidth;
                    }
                } else {
                    if (height > maxHeight) {
                        width *= maxHeight / height;
                        height = maxHeight;
                    }
                }
                canvas.width = width;
                canvas.height = height;
                ctx.drawImage(img, 0, 0, width, height);
                const imageData = canvas.toDataURL('image/jpeg', 0.03).split(',')[1]; // Lower quality to 0.03
                log('Compressed image captured, Base64 length: ' + imageData.length);
                resolve(imageData);
            };
            img.onerror = reject;
            img.src = reader.result;
        };
        reader.onerror = reject;
        reader.readAsDataURL(blob);
    });
}

let ws;
let messageQueue = [];

function sendToHyperion(imageData) {
    const jsonData = {
        command: "image",
        imagedata: imageData,
        name: "TizenScreenCapture",
        format: "auto",
        origin: "TizenApp",
        priority: 50,
        duration: 30000 // Duración de 30 segundos
    };

    log('Sending screenshot to Hyperion');

    if (ws.readyState === WebSocket.OPEN) {
        ws.send(JSON.stringify(jsonData));
    } else {
        messageQueue.push(JSON.stringify(jsonData));
    }
}

function initializeWebSocket() {
    const ipHyperion = '192.168.88.101';
    const puertoHyperion = 8090;

    ws = new WebSocket(`ws://${ipHyperion}:${puertoHyperion}/jsonrpc`);

    ws.onopen = function () {
        log('WebSocket connection opened');
        while (messageQueue.length > 0) {
            ws.send(messageQueue.shift());
        }
    };

    ws.onmessage = function (event) {
        log('Message from server: ' + event.data);
    };

    ws.onerror = function (error) {
        log('WebSocket error: ' + error.message);
    };

    ws.onclose = function () {
        log('WebSocket connection closed');
        // Intentar reconectar después de 1 segundo
        setTimeout(initializeWebSocket, 1000);
    };
}

function sendRandomColor() {
    const ipHyperion = '192.168.88.101';
    const puertoHyperion = 8090;

    const colors = [
        [255, 0, 0],    // Red
        [0, 255, 0],    // Green
        [0, 0, 255],    // Blue
        [255, 255, 0],  // Yellow
        [255, 0, 255],  // Magenta
        [0, 255, 255]   // Cyan
    ];

    const randomColor = colors[Math.floor(Math.random() * colors.length)];

    const jsonData = {
        command: "color",
        color: randomColor,
        duration: 120000,  // 2 minutes
        priority: 20,
        origin: "TizenApp"
    };

    log('Sending color to Hyperion: ' + JSON.stringify(jsonData));

    fetch(`http://${ipHyperion}:${puertoHyperion}/json-rpc`, {
        method: 'POST',
        headers: {
            'Content-Type': 'application/json'
        },
        body: JSON.stringify(jsonData)
    })
    .then(response => response.json())
    .then(data => log('Response from Hyperion: ' + JSON.stringify(data)))
    .catch(error => log('Error sending color to Hyperion: ' + error));
}

// Configuración del reproductor de YouTube
let player;
function onYouTubeIframeAPIReady() {
    player = new YT.Player('player', {
        height: '390',
        width: '640',
        videoId: 'Gt6wKDnG0xA',
        events: {
            'onReady': onPlayerReady,
            'onStateChange': onPlayerStateChange
        }
    });
}

let captureInterval;
function startVideoCapture() {
    if (!player || !player.getIframe) {
        log('YouTube Player not ready');
        return;
    }

    log('Starting video capture');
    const iframe = player.getIframe();
    const videoElement = iframe.contentWindow.document.querySelector('video');
    if (!videoElement) {
        log('Video element not found');
        return;
    }

    const canvas = document.createElement('canvas');
    const context = canvas.getContext('2d');

    canvas.width = 120;
    canvas.height = 120;

    captureInterval = setInterval(() => {
        try {
            context.drawImage(videoElement, 0, 0, canvas.width, canvas.height);
            const imageData = canvas.toDataURL('image/jpeg', 0.03).split(',')[1]; // Lower quality to 0.03
            sendToHyperion(imageData);
        } catch (error) {
            log('Error capturing video frame: ' + error);
            clearInterval(captureInterval);
        }
    }, 100); // Capturar cada 100 ms
}

function stopVideoCapture() {
    log('Stopping video capture');
    clearInterval(captureInterval);
}

function onPlayerReady(event) {
    event.target.playVideo();
}

function onPlayerStateChange(event) {
    if (event.data == YT.PlayerState.PLAYING) {
        log('YouTube Player playing');
        startVideoCapture();
    } else {
        log('YouTube Player paused or ended');
        stopVideoCapture();
    }
}

function log(message) {
    const logs = document.getElementById('logs');
    const logMessage = document.createElement('p');
    logMessage.textContent = message;
    logs.appendChild(logMessage);
    console.log(message);
}

index.html：

<!DOCTYPE html>
<html>
<head>
    <title>SamyGrabber</title>
    <style>
        body {
            font-family: Arial, sans-serif;
            background-color: #121212;
            color: white;
            text-align: center;
            padding: 20px;
        }
        #captureArea {
            width: 100%;
            height: 300px;
            border: 1px solid #ccc;
            margin-bottom: 20px;
            display: flex;
            justify-content: center;
            align-items: center;
        }
        #logs {
            background-color: #222;
            padding: 10px;
            border-radius: 5px;
            margin-top: 20px;
            max-height: 200px;
            overflow-y: scroll;
            text-align: left;
        }
        .button-container {
            display: flex;
            justify-content: center;
            gap: 20px;
        }
        button {
            padding: 10px 20px;
            font-size: 16px;
            border: none;
            border-radius: 5px;
            background-color: #6200ee;
            color: white;
            cursor: pointer;
        }
        button:hover {
            background-color: #3700b3;
        }
    </style>
</head>
<body>
    <h1>SamyGrabber</h1>
    <div id="captureArea" contenteditable="true">
        <span>Texto</span>
    </div>
    <div class="button-container">
        <button id="startCapture" tabindex="1">Iniciar Captura</button>
        <button id="remoteButton" tabindex="2">Captura con Control Remoto</button>
        <button id="testImageButton" tabindex="3">Enviar Imagen de Prueba</button>
        <button id="sendColorButton" tabindex="4">Enviar Color Aleatorio</button>
        <button id="startVideoCaptureButton" tabindex="5">Iniciar Captura de Video</button>
        <button id="stopVideoCaptureButton" tabindex="6">Detener Captura de Video</button>
    </div>
    <div id="logs"></div>
    <div id="player"></div>
    <script src="https://www.youtube.com/iframe_api"></script>
    <script src="main.js"></script>
</body>
</html>

挑战：

1. Web 应用程序限制： 当前的解决方案仅限于捕获 Web 应用程序中的内容。我的目标是捕获电视的整个屏幕内容，包括当网络应用程序未聚焦或运行时。
2. 权限问题： 当尝试使用某些 API（例如 captureScreen）时，我遇到权限错误。尽管在 config.xml 文件中设置了必要的权限，但这仍然存在。
3. 替代解决方案： 我正在探索替代方法，例如使用低级图形 API（OpenGL、Vulkan）或其他本机功能来实现屏幕捕获。然而，我们将非常感谢有关最佳开始方法的指导。 目标： • 我的目标是实时捕获电视屏幕边缘的颜色信息，无论正在运行的应用程序或电视状态如何。 • 解决方案应连续、高效地工作，无需用户持续干预。 问题：
4. 屏幕捕获的最佳方法： 在 Tizen TV 上捕获整个屏幕内容的推荐方法是什么？是否有我需要注意的特定 API 或权限？
5. 使用低级图形 API： 使用 OpenGL 或 Vulkan 捕获屏幕内容的可行性如何？有任何示例或文档可以帮助我入门吗？
6. 处理权限： 尝试访问屏幕内容时如何克服权限问题？是否需要配置特定设置或权限？

任何指导、文档或示例都会非常有帮助。预先感谢您的协助！

Answer 1

1. 一般来说，第三方公司是不允许在后台使用JavaScript的。允许在后台执行 JavaScript 行为需要非常高的权限，如果绝对需要此功能，您将需要联系您的应用程序管理员寻求帮助。

2. 您似乎正在尝试捕获应用程序中存在的画布。从您的问题来看，您似乎还想捕获应用程序之外的内容。但是，由于政策限制，应用程序无法捕获自身外部的任何内容。不支持此功能。