let socket = null; let reconnectInterval = null; const bodyConnectionStatusDom = document.getElementById('bodyConnectionStatus'); const bodyVideoFeedDom = document.getElementById('bodyVideoFeed'); const bodyHandStatusDom = document.getElementById('bodyHandStatus'); let isClickBth = false; let mediaStream = null; let videoElement = null; let canvasElement = null; let canvasContext = null; let captureInterval = null; // 远程服务器配置 - 直接连接到远程服务器 const REMOTE_SERVER_CONFIG = { // 修改为您的远程服务器地址 host: '103.8.33.232', // 远程服务器IP port: 8080, // WebSocket端口 protocol: 'ws', // 或 'wss' 如果使用SSL reconnectDelay: 5000, // 重连延迟 heartbeatInterval: 30000, // 心跳间隔 maxReconnectAttempts: 10 // 最大重连次数 }; let reconnectAttempts = 0; let heartbeatTimer = null; let connectionStartTime = null; let totalFramesSent = 0; let totalProcessingTime = 0; function getWebSocketUrl() { return `${REMOTE_SERVER_CONFIG.protocol}://${REMOTE_SERVER_CONFIG.host}:${REMOTE_SERVER_CONFIG.port}`; } function connectWebSocket() { try { const wsUrl = getWebSocketUrl(); console.log(`正在连接到远程服务器: ${wsUrl}`); // 连接到远程服务器 socket = new WebSocket(wsUrl); connectionStartTime = Date.now(); socket.onopen = function (event) { bodyConnectionStatusDom.textContent = '已直接连接到远程体感检测服务器'; bodyConnectionStatusDom.className = 'status connected'; clearInterval(reconnectInterval); reconnectAttempts = 0; console.log('成功连接到远程服务器'); // 启动心跳 startHeartbeat(); // 开始捕获摄像头 startCameraCapture(); // 网页间通信 htmlPostMessage( { type: "体感识别", status: "start", data: "", } ); }; // 上一次发送的消息 let preStatusText = ""; socket.onmessage = function (event) { const data = JSON.parse(event.data); // 处理连接确认消息 if (data.success && data.message && data.client_id) { console.log('连接成功:', data.message); console.log('客户端ID:', data.client_id); console.log('服务器信息:', data.server_info); // 更新连接状态显示 if (data.server_info) { const connInfo = `${data.server_info.current_connections}/${data.server_info.max_connections}`; bodyConnectionStatusDom.textContent = `已连接到远程服务器 (${connInfo})`; } return; } // 处理心跳响应 if (data.type === 'pong') { console.log('收到服务器心跳响应,服务器状态:', data.server_status); updateServerStatus(data); return; } // 处理图像处理结果 if (data.success && data.image) { // 显示处理后的图像(包含美化的关键点和骨骼线) // 创建镜像效果 const img = new Image(); img.onload = function() { const canvas = document.createElement('canvas'); const ctx = canvas.getContext('2d'); // 设置画布大小 canvas.width = img.width; canvas.height = img.height; // 水平翻转(镜像效果) ctx.scale(-1, 1); ctx.drawImage(img, -canvas.width, 0); // 转换为base64并显示 bodyVideoFeedDom.src = canvas.toDataURL('image/jpeg', 0.8); }; img.src = 'data:image/jpeg;base64,' + data.image; // 更新统计信息 if (data.processing_time) { totalFramesSent++; totalProcessingTime += data.processing_time; updatePerformanceStats(data.processing_time); } // 处理检测结果 if (data.detections && data.detections.length > 0) { let statusText = ''; let hasHandRaised = false; let allFocused = true; let handRaisedDetails = []; data.detections.forEach((detection, index) => { if (detection.hand_raised) { hasHandRaised = true; handRaisedDetails.push({ id: detection.id, hand_status: detection.hand_status }); } if (!detection.attention_focused) { allFocused = false; } }); // 构建简化的状态文本 if (hasHandRaised) { const firstHandRaised = handRaisedDetails[0]; if (firstHandRaised && firstHandRaised.hand_status !== 'none') { let hand = null; switch (firstHandRaised.hand_status) { case 'left': hand = 'left'; statusText = '左手举起'; break; case 'right': hand = 'right'; statusText = '右手举起'; break; case 'double': hand = 'left'; // 默认选择左手 statusText = '双手举起'; break; } if (hand) { console.log(`🎯 检测到${hand === 'left' ? '左手' : '右手'}举起,准备执行点击`); // 在所有页面都允许体感交互点击 clickBth(hand); } } } else { statusText = '未举手'; } // 添加注意力状态 if (allFocused) { statusText += ' | 注意力集中'; } else { // 检查是否有人在低头 let hasLookingDown = false; data.detections.forEach(detection => { if (detection.attention_reason === 'looking_down') { hasLookingDown = true; } }); if (hasLookingDown) { statusText += ' | 正在低头'; } else { statusText += ' | 注意力分散'; } } // 添加检测到的人数信息 if (data.total_persons !== undefined) { statusText += ` | 检测到${data.total_persons}人`; } // console.log("preStatusText statusText", preStatusText, statusText, preStatusText !== statusText); if (preStatusText !== statusText) { preStatusText = statusText; // 网页间通信 htmlPostMessage( { type: "体感识别", status: "process", data: statusText, } ); } bodyHandStatusDom.textContent = statusText; } else { bodyHandStatusDom.textContent = '未检测到人'; } } else if (data.success === false && data.error) { console.error('服务器处理错误:', data.error); bodyHandStatusDom.textContent = '检测错误: ' + data.error; // 如果是队列满了,稍后重试 if (data.error.includes('队列已满') || data.error.includes('queue is full')) { console.log('处理队列已满,3秒后重试...'); setTimeout(() => { if (socket && socket.readyState === WebSocket.OPEN) { console.log('重新开始发送数据'); } }, 3000); } // 如果是连接数超限,显示特殊提示 if (data.error.includes('连接数已达上限') || data.error.includes('connection limit')) { bodyConnectionStatusDom.textContent = '服务器连接数已满,请稍后重试'; bodyConnectionStatusDom.className = 'status error'; } } }; socket.onclose = function (event) { console.log('WebSocket连接关闭,代码:', event.code, '原因:', event.reason); bodyConnectionStatusDom.textContent = '与远程服务器的连接已断开,尝试重新连接...'; bodyConnectionStatusDom.className = 'status disconnected'; // 停止心跳 stopHeartbeat(); // 停止摄像头捕获 stopCameraCapture(); // 设置重连 if (reconnectAttempts < REMOTE_SERVER_CONFIG.maxReconnectAttempts) { if (!reconnectInterval) { reconnectInterval = setInterval(() => { reconnectAttempts++; console.log(`第${reconnectAttempts}次重连尝试...`); connectWebSocket(); }, REMOTE_SERVER_CONFIG.reconnectDelay); } } else { console.error('达到最大重连次数,停止重连'); bodyConnectionStatusDom.textContent = '连接失败,请刷新页面重试'; bodyConnectionStatusDom.className = 'status error'; } // 网页间通信 htmlPostMessage( { type: "体感识别", status: "end", data: "", } ); }; socket.onerror = function (error) { console.error('WebSocket错误:', error); bodyConnectionStatusDom.textContent = '远程连接错误'; bodyConnectionStatusDom.className = 'status disconnected'; }; } catch (e) { console.error('创建WebSocket时出错:', e); bodyConnectionStatusDom.textContent = '连接错误: ' + e.message; bodyConnectionStatusDom.className = 'status disconnected'; } } function startHeartbeat() { stopHeartbeat(); // 确保没有重复的心跳 heartbeatTimer = setInterval(() => { if (socket && socket.readyState === WebSocket.OPEN) { socket.send(JSON.stringify({ type: 'ping', timestamp: Date.now() })); } }, REMOTE_SERVER_CONFIG.heartbeatInterval); } function stopHeartbeat() { if (heartbeatTimer) { clearInterval(heartbeatTimer); heartbeatTimer = null; } } function updateServerStatus(data) { // 更新服务器状态信息 if (data.queue_size !== undefined && data.connections !== undefined) { const statusInfo = `队列: ${data.queue_size}, 连接: ${data.connections}`; console.log('服务器状态:', statusInfo); } } function updatePerformanceStats(processingTime) { // 计算平均处理时间 const avgProcessingTime = totalProcessingTime / totalFramesSent; const sessionDuration = (Date.now() - connectionStartTime) / 1000; // console.log(`性能统计 - 已发送帧数: ${totalFramesSent}, 平均处理时间: ${(avgProcessingTime * 1000).toFixed(0)}ms, 会话时长: ${sessionDuration.toFixed(0)}s`); } function startCameraCapture() { // 创建视频元素用于捕获摄像头 if (!videoElement) { videoElement = document.createElement('video'); videoElement.autoplay = true; videoElement.style.display = 'none'; document.body.appendChild(videoElement); } // 创建画布用于截取帧 if (!canvasElement) { canvasElement = document.createElement('canvas'); canvasElement.style.display = 'none'; document.body.appendChild(canvasElement); canvasContext = canvasElement.getContext('2d'); } // 获取摄像头权限 navigator.mediaDevices.getUserMedia({ video: { // width: { ideal: 640 }, // height: { ideal: 480 }, width: { ideal: 960 }, height: { ideal: 560 }, // frameRate: { ideal: 60, max: 120 } // 限制帧率以减少服务器压力 } }) .then(stream => { mediaStream = stream; videoElement.srcObject = stream; // 设置画布大小 videoElement.onloadedmetadata = () => { canvasElement.width = videoElement.videoWidth; canvasElement.height = videoElement.videoHeight; // 开始定期捕获和发送帧 - 适当降低频率 // captureInterval = setInterval(captureAndSendFrame, 1500); // 每1秒捕获一次 captureInterval = setInterval(captureAndSendFrame, 500); console.log('摄像头初始化完成,开始远程体感检测'); console.log(`视频分辨率: ${videoElement.videoWidth}x${videoElement.videoHeight}`); }; }) .catch(error => { console.error('获取摄像头权限失败:', error); bodyHandStatusDom.textContent = '无法访问摄像头: ' + error.message; }); } function captureAndSendFrame() { if (!videoElement || !canvasContext || !socket || socket.readyState !== WebSocket.OPEN) { return; } try { // 在画布上绘制当前视频帧 canvasContext.drawImage(videoElement, 0, 0, canvasElement.width, canvasElement.height); // 转换为base64 canvasElement.toBlob(blob => { if (!blob) { console.error('无法创建图像blob'); return; } const reader = new FileReader(); reader.onloadend = () => { try { const base64data = reader.result.split(',')[1]; // 发送到远程服务器 if (socket && socket.readyState === WebSocket.OPEN) { const message = { type: 'image', image: base64data, timestamp: Date.now(), client_info: { frame_number: totalFramesSent + 1, session_duration: Date.now() - connectionStartTime } }; socket.send(JSON.stringify(message)); // console.log(`发送第${totalFramesSent + 1}帧到远程服务器`); } } catch (e) { console.error('发送图像数据时出错:', e); } }; reader.onerror = () => { console.error('读取图像数据时出错'); }; reader.readAsDataURL(blob); }, 'image/jpeg', 0.7); // 降低质量以减少数据量 } catch (e) { console.error('捕获图像帧时出错:', e); } } function stopCameraCapture() { // 停止定期捕获 if (captureInterval) { clearInterval(captureInterval); captureInterval = null; } // 停止媒体流 if (mediaStream) { mediaStream.getTracks().forEach(track => track.stop()); mediaStream = null; } // 清理视频元素 if (videoElement) { videoElement.srcObject = null; } reconnectAttempts = 10; console.log('摄像头捕获已停止'); } function clickBth(hand) { console.log(`🎯 体感交互触发: ${hand} 手举起`); if (isClickBth) { console.log("⏳ 正在处理中,忽略重复触发"); return; } isClickBth = true; // 语音广播(如果函数存在) if (typeof voiceBroadcast === 'function') { voiceBroadcast(`您举起${hand == 'left' ? '左' : '右'}手,正在帮您选择${hand == 'left' ? '左' : '右'}边选项!`); } else { console.log(`🔊 您举起${hand == 'left' ? '左' : '右'}手,正在帮您选择${hand == 'left' ? '左' : '右'}边选项!`); } // 立即执行按钮检测和点击 setTimeout(() => { try { // 首先检查消息弹窗状态 const messageDialog = document.getElementById('message'); if (messageDialog && messageDialog.classList.contains('show')) { console.log("🚫 消息弹窗正在显示,点击确认按钮"); const messageBtn = document.getElementById('messageBtn'); if (messageBtn) { messageBtn.click(); console.log("✅ 点击了消息确认按钮"); } setTimeout(() => { isClickBth = false; }, 1000); return; } // 增强的按钮检测逻辑 let btnLeftDom = null; let btnRightDom = null; let detectionMethod = ""; // 方法1: 检查比大小游戏的标准按钮ID btnLeftDom = document.getElementById("leftBtn"); btnRightDom = document.getElementById("rightBtn"); if (btnLeftDom && btnRightDom) { detectionMethod = "标准游戏按钮ID"; } // 方法2: 按class查找 if (!btnLeftDom || !btnRightDom) { btnLeftDom = document.getElementsByClassName("box-btn-left")[0]; btnRightDom = document.getElementsByClassName("box-btn-right")[0]; if (btnLeftDom && btnRightDom) { detectionMethod = "box-btn class"; } } // 方法3: 查找所有button元素,按位置判断 if (!btnLeftDom || !btnRightDom) { const allButtons = document.querySelectorAll("button"); console.log("🔍 找到的所有button元素:", allButtons); if (allButtons.length >= 2) { // 按照页面位置排序(从左到右) const sortedButtons = Array.from(allButtons).sort((a, b) => { const rectA = a.getBoundingClientRect(); const rectB = b.getBoundingClientRect(); return rectA.left - rectB.left; }); btnLeftDom = sortedButtons[0]; btnRightDom = sortedButtons[1]; detectionMethod = "按位置排序的button元素"; console.log("🔍 按位置排序的按钮:", { left: { element: btnLeftDom, text: btnLeftDom.textContent, position: btnLeftDom.getBoundingClientRect() }, right: { element: btnRightDom, text: btnRightDom.textContent, position: btnRightDom.getBoundingClientRect() } }); } } // 方法4: 查找有onclick事件的元素 if (!btnLeftDom || !btnRightDom) { const clickableElements = Array.from(document.querySelectorAll("*")).filter(el => { return el.onclick || el.addEventListener || el.hasAttribute('onclick'); }); console.log("🔍 找到的可点击元素:", clickableElements); if (clickableElements.length >= 2) { const sortedClickable = clickableElements.sort((a, b) => { const rectA = a.getBoundingClientRect(); const rectB = b.getBoundingClientRect(); return rectA.left - rectB.left; }); btnLeftDom = sortedClickable[0]; btnRightDom = sortedClickable[1]; detectionMethod = "可点击元素"; } } // 方法5: 智能文本匹配 if (!btnLeftDom || !btnRightDom) { const allElements = document.querySelectorAll("*"); const leftKeywords = ["左", "left", "选择", "A", "1"]; const rightKeywords = ["右", "right", "选择", "B", "2"]; let leftCandidates = []; let rightCandidates = []; allElements.forEach(el => { const text = el.textContent.toLowerCase(); const isClickable = el.onclick || el.tagName === 'BUTTON' || el.hasAttribute('onclick'); if (isClickable) { if (leftKeywords.some(keyword => text.includes(keyword.toLowerCase()))) { leftCandidates.push(el); } if (rightKeywords.some(keyword => text.includes(keyword.toLowerCase()))) { rightCandidates.push(el); } } }); if (leftCandidates.length > 0) btnLeftDom = leftCandidates[0]; if (rightCandidates.length > 0) btnRightDom = rightCandidates[0]; if (btnLeftDom && btnRightDom) detectionMethod = "智能文本匹配"; } console.log("🔍 最终按钮检测结果:", { method: detectionMethod, leftBtn: btnLeftDom ? { tag: btnLeftDom.tagName, id: btnLeftDom.id, class: btnLeftDom.className, text: btnLeftDom.textContent.trim(), disabled: btnLeftDom.disabled } : "未找到", rightBtn: btnRightDom ? { tag: btnRightDom.tagName, id: btnRightDom.id, class: btnRightDom.className, text: btnRightDom.textContent.trim(), disabled: btnRightDom.disabled } : "未找到" }); // 执行点击操作 let targetBtn = null; let btnName = ""; if (hand === 'left' && btnLeftDom) { targetBtn = btnLeftDom; btnName = "左手按钮"; } else if (hand === 'right' && btnRightDom) { targetBtn = btnRightDom; btnName = "右手按钮"; } if (targetBtn) { console.log(`🎯 准备点击${btnName}:`, targetBtn); // 检查按钮是否可点击 if (targetBtn.disabled) { console.warn(`⚠️ ${btnName}已禁用,无法点击`); return; } // 尝试多种点击方式,增加成功率 let clickSuccess = false; try { // 方法1: 模拟真实的鼠标点击事件 const rect = targetBtn.getBoundingClientRect(); const clickEvent = new MouseEvent('click', { bubbles: true, cancelable: true, view: window, clientX: rect.left + rect.width / 2, clientY: rect.top + rect.height / 2 }); // 先触发mousedown和mouseup事件 targetBtn.dispatchEvent(new MouseEvent('mousedown', clickEvent)); targetBtn.dispatchEvent(new MouseEvent('mouseup', clickEvent)); targetBtn.dispatchEvent(clickEvent); console.log(`✅ 方法1成功: 完整鼠标事件模拟 ${btnName}`); clickSuccess = true; } catch (e1) { console.warn(`❌ 方法1失败:`, e1); try { // 方法2: 直接click() targetBtn.click(); console.log(`✅ 方法2成功: 直接click() ${btnName}`); clickSuccess = true; } catch (e2) { console.warn(`❌ 方法2失败:`, e2); try { // 方法3: 触发onclick函数 if (targetBtn.onclick) { targetBtn.onclick(); console.log(`✅ 方法3成功: onclick函数 ${btnName}`); clickSuccess = true; } } catch (e3) { console.error(`❌ 方法3失败:`, e3); } } } if (!clickSuccess) { console.error(`❌ 所有点击方法都失败了,按钮信息:`, { element: targetBtn, tag: targetBtn.tagName, id: targetBtn.id, class: targetBtn.className, onclick: !!targetBtn.onclick, disabled: targetBtn.disabled }); } // 添加视觉反馈 if (clickSuccess) { targetBtn.style.transform = 'scale(1.1)'; targetBtn.style.transition = 'transform 0.2s'; targetBtn.style.backgroundColor = '#4CAF50'; setTimeout(() => { targetBtn.style.transform = ''; targetBtn.style.backgroundColor = ''; }, 300); } } else { console.error(`❌ 未找到对应的${hand === 'left' ? '左' : '右'}手按钮`); // 详细的调试信息 console.log("🔍 详细调试信息:"); console.log("- 当前页面URL:", window.location.href); console.log("- 页面标题:", document.title); console.log("- 所有button元素:", document.querySelectorAll("button")); console.log("- 所有有id的元素:", Array.from(document.querySelectorAll("[id]")).map(el => ({id: el.id, tag: el.tagName}))); console.log("- 所有有class的元素:", Array.from(document.querySelectorAll("[class]")).map(el => ({class: el.className, tag: el.tagName}))); } // 隐藏体感交互容器 let bodyDom = document.getElementsByClassName("box-body")[0]; if (bodyDom) { bodyDom.style.display = "none"; } } catch (e) { console.error('🚨 执行按钮点击时出错:', e); console.error('错误堆栈:', e.stack); } // 重置点击状态 setTimeout(() => { isClickBth = false; console.log("🔄 重置点击状态,允许下次检测"); }, 1500); }, 500); } // 添加页面按钮实时检测函数 window.detectCurrentPageButtons = function() { console.log("🔍 检测当前页面的所有可交互元素:"); const info = { url: window.location.href, title: document.title, buttons: Array.from(document.querySelectorAll("button")).map(btn => ({ tag: btn.tagName, id: btn.id, className: btn.className, text: btn.textContent.trim(), disabled: btn.disabled, position: btn.getBoundingClientRect() })), clickableElements: Array.from(document.querySelectorAll("*")).filter(el => el.onclick || el.hasAttribute('onclick') ).map(el => ({ tag: el.tagName, id: el.id, className: el.className, text: el.textContent.trim().substring(0, 50), position: el.getBoundingClientRect() })) }; console.log("页面信息:", info); return info; }; // 打开体感交互 function showBody() { let bodyDom = document.getElementsByClassName("box-body")[0]; bodyDom.style.display = bodyDom.style.display == 'none' ? 'block' : 'none'; if (bodyDom.style.display == 'block') { // 重置统计信息 totalFramesSent = 0; totalProcessingTime = 0; reconnectAttempts = 0; // 初始连接到远程服务器 connectWebSocket(); isClickBth = false; console.log('启动直接远程体感检测系统'); console.log('服务器地址:', getWebSocketUrl()); voiceBroadcast("开启体感交互"); } else { // 停止摄像头 stopCameraCapture(); // 停止心跳 stopHeartbeat(); // 关闭WebSocket连接 if (socket) { socket.close(1000, "正常关闭"); } // 清除重连定时器 if (reconnectInterval) { clearInterval(reconnectInterval); reconnectInterval = null; } console.log('关闭直接远程体感检测系统'); voiceBroadcast("关闭体感交互"); // 显示会话统计 if (connectionStartTime) { const sessionDuration = (Date.now() - connectionStartTime) / 1000; const avgProcessingTime = totalProcessingTime / Math.max(totalFramesSent, 1); console.log(`会话结束 - 时长: ${sessionDuration.toFixed(0)}s, 处理帧数: ${totalFramesSent}, 平均处理时间: ${(avgProcessingTime * 1000).toFixed(0)}ms`); } } } // 添加页面卸载时的清理 window.addEventListener('beforeunload', function () { if (socket) { socket.close(1000, "页面关闭"); } stopCameraCapture(); stopHeartbeat(); }); // 添加网络状态监听 window.addEventListener('online', function () { console.log('网络已连接'); if (!socket || socket.readyState !== WebSocket.OPEN) { console.log('网络恢复,尝试重新连接...'); connectWebSocket(); } }); window.addEventListener('offline', function () { console.log('网络已断开'); bodyConnectionStatusDom.textContent = '网络连接已断开'; bodyConnectionStatusDom.className = 'status disconnected'; }); // 全局函数,供外部调用 window.startBodySensation = startBodySensation; window.stopBodySensation = stopBodySensation; window.showBody = showBody; window.hideBody = hideBody; window.clickBth = clickBth; // 测试函数 window.testButtonDetection = function() { console.log("🧪 开始测试按钮检测..."); console.log("📋 当前页面URL:", window.location.href); console.log("📋 当前页面标题:", document.title); // 测试各种按钮选择器 console.log("🔍 测试courseHome按钮:"); console.log("- box-btn-left:", document.getElementsByClassName("box-btn-left")); console.log("- box-btn-right:", document.getElementsByClassName("box-btn-right")); console.log("🔍 测试比大小游戏按钮:"); console.log("- leftBtn:", document.getElementById("leftBtn")); console.log("- rightBtn:", document.getElementById("rightBtn")); console.log("🔍 测试示例游戏按钮:"); console.log("- .option-btn:", document.querySelectorAll(".option-btn")); console.log("🔍 测试AI游戏按钮:"); console.log("- .ai-option-btn:", document.querySelectorAll(".ai-option-btn")); // 测试点击功能 console.log("🧪 测试左手点击..."); clickBth('left'); setTimeout(() => { console.log("🧪 测试右手点击..."); clickBth('right'); }, 1000); };