回答-BFW问答

可以通过face-api通过摄像头捕获人脸表情数据控制vrm三维人物表情，效果如下：

800_auto

完整代码如下：

点击查看全文

<!DOCTYPE html>
<html>
<head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no" />
    <style>
        body { margin: 0; }
        canvas { display: block; }
        #video { position: absolute; top: 10px; right: 10px; width: 320px; height: 240px; }
    </style>
</head>
<body>
    <video id="video" width="320" height="240" autoplay muted></video>

<script type="text/javascript" src="//repo.bfw.wiki/bfwrepo/js/face-api.1.7.13.js"></script>
    <script type="importmap">
    {
        "imports": {
            "three": "//repo.bfw.wiki/bfwrepo/js/module/three/build/164/three.module.js",
            "three/addons/": "//repo.bfw.wiki/bfwrepo/js/module/three/examples/164/jsm/",
            "@pixiv/three-vrm": "//repo.bfw.wiki/bfwrepo/js/three-vrm.module.3.0.0.js"
        }
    }
    </script>

    <script type="module">
        import * as THREE from 'three';
        import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
        import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
        import { VRMLoaderPlugin, VRMUtils } from '@pixiv/three-vrm';

        let currentVrm, faceDetection;
const modelPath = '../models/'; // path to model folder that will be loaded using http

const minScore = 0.2; // minimum score
const maxResults = 5; // maximum number of results to return
let optionsSSDMobileNet;
  // 尝试获取第三个摄像头（如果存在的话）
            const devices = await navigator.mediaDevices.enumerateDevices();
            const videoDevices = devices.filter(device => device.kind === 'videoinput');
            const deviceId = videoDevices.length >= 1 ? videoDevices[0].deviceId : undefined;

            const stream = await navigator.mediaDevices.getUserMedia({
                video: deviceId ? { deviceId: { exact: deviceId } } : true
            });
            video.srcObject = stream;
async function detectVideo() {
 
  const t0 = performance.now();
  faceapi
    .detectAllFaces(video, optionsSSDMobileNet)
    .withFaceLandmarks()
    .withFaceExpressions()
    // .withFaceDescriptors()
    .withAgeAndGender()
    .then((result) => {
      const fps = 1000 / (performance.now() - t0);
     console.log(result[0]);
      
     
    updateVRMExpressions(result[0]);
     requestAnimationFrame(() => detectVideo());
      return true;
    })
    .catch((err) => {
      console.log(err);
       requestAnimationFrame(() => detectVideo());
      return true;
    });
  return false;
}

async function setupFaceAPI() {
  // load face-api models
  // log('Models loading');
  // await faceapi.nets.tinyFaceDetector.load(modelPath); // using ssdMobilenetv1
  await faceapi.nets.ssdMobilenetv1.load(modelPath);
  await faceapi.nets.ageGenderNet.load(modelPath);
  await faceapi.nets.faceLandmark68Net.load(modelPath);
  await faceapi.nets.faceRecognitionNet.load(modelPath);
  await faceapi.nets.faceExpressionNet.load(modelPath);
  optionsSSDMobileNet = new faceapi.SsdMobilenetv1Options({ minConfidence: minScore, maxResults });
  // check tf engine state

}

        // 初始化face-api
        async function initVrm() {
    	// renderer
			const renderer = new THREE.WebGLRenderer();
			renderer.setSize( window.innerWidth, window.innerHeight );
			renderer.setPixelRatio( window.devicePixelRatio );
			document.body.appendChild( renderer.domElement );

			// camera
			const camera = new THREE.PerspectiveCamera( 30.0, window.innerWidth / window.innerHeight, 0.1, 20.0 );
			camera.position.set( 0.0, 1.0, 5.0 );

			// camera controls
			const controls = new OrbitControls( camera, renderer.domElement );
			controls.screenSpacePanning = true;
			controls.target.set( 0.0, 1.0, 0.0 );
			controls.update();

			// scene
			const scene = new THREE.Scene();

			// light
			const light = new THREE.DirectionalLight( 0xffffff, Math.PI );
			light.position.set( 1.0, 1.0, 1.0 ).normalize();
			scene.add( light );

			// gltf and vrm
			const loader = new GLTFLoader();
			loader.crossOrigin = 'anonymous';
			loader.register( ( parser ) => new VRMLoaderPlugin( parser ) );

			loader.load(
				'//repo.bfw.wiki/bfwrepo/threemodel/girl.vrm',
				( gltf ) => {
					const vrm = gltf.userData.vrm;
					VRMUtils.removeUnnecessaryVertices( gltf.scene );
					VRMUtils.removeUnnecessaryJoints( gltf.scene );
					vrm.scene.traverse( ( obj ) => {
						obj.frustumCulled = false;
					} );
					scene.add( vrm.scene );
					currentVrm = vrm;
					console.log( vrm );
				},
				( progress ) => console.log( 'Loading model...', 100.0 * ( progress.loaded / progress.total ), '%' ),
				( error ) => console.error( error )
			);

			// helpers
			const gridHelper = new THREE.GridHelper( 10, 10 );
			scene.add( gridHelper );

			const axesHelper = new THREE.AxesHelper( 5 );
			scene.add( axesHelper );

			// animate
			const clock = new THREE.Clock();

			function animate() {
				requestAnimationFrame( animate );
				const deltaTime = clock.getDelta();
				if ( currentVrm ) {
					currentVrm.update( deltaTime );
				}
				renderer.render( scene, camera );
			}

			animate();

			// Expression control functions
			window.setExpression = function(expression) {
				if (currentVrm) {
resetAllExpressions();


					switch(expression) {
						case 'angry':
							currentVrm.expressionManager.setValue('angry', 1.0);
							break;
						case 'happy':
							currentVrm.expressionManager.setValue('happy', 1.0);
							break;
						case 'sad':
							currentVrm.expressionManager.setValue('sad', 1.0);
							break;
						case 'neutral':
							currentVrm.expressionManager.setValue('relaxed', 1.0);
							break;
								case 'fearful':
							currentVrm.expressionManager.setValue('fearful', 1.0);
							break;
						case 'crying':
							currentVrm.expressionManager.setValue('sad', 1.0);
							currentVrm.expressionManager.setValue('aa', 0.8); // Open mouth
							break;
							case 'surprised':
							currentVrm.expressionManager.setValue('surprised', 1.0);
							currentVrm.expressionManager.setValue('aa', 0.8); // Open mouth
							break;
					}
				}
        }
        }

        	function resetAllExpressions() {
				const expressions = ['angry', 'happy', 'sad', 'disgusted', 'relaxed',"surprised","fearful"];
				expressions.forEach(exp => {
					currentVrm.expressionManager.setValue(exp, 0);
				});
			}
		
         // 更新VRM表情
        function updateVRMExpressions(exp) {
                   
            let expressions=exp.expressions;
            console.log(expressions)
     

        if (!currentVrm) return;
 
        setExpression(findMaxProperty(expressions))
       
 

           
        }
function findMaxProperty(obj) {
  let maxProp = null;
  let maxValue = -Infinity;

  for (const prop in obj) {
    if (obj[prop] > maxValue) {
      maxValue = obj[prop];
      maxProp = prop;
    }
  }

  return maxProp;
}

        // 初始化并开始动画
        async function init() {
            
             
              await faceapi.tf.setBackend('webgl');
              await faceapi.tf.ready();
            
              // tfjs optimizations
              if (faceapi.tf?.env().flagRegistry.CANVAS2D_WILL_READ_FREQUENTLY) faceapi.tf.env().set('CANVAS2D_WILL_READ_FREQUENTLY', true);
              if (faceapi.tf?.env().flagRegistry.WEBGL_EXP_CONV) faceapi.tf.env().set('WEBGL_EXP_CONV', true);
              if (faceapi.tf?.env().flagRegistry.WEBGL_EXP_CONV) faceapi.tf.env().set('WEBGL_EXP_CONV', true);
       
            
              await setupFaceAPI();
    

           await detectVideo();
           await initVrm();
            //animate();
        }

        init();
    </script>
</body>
</html>

VRM模型中的表情通常使用blendshapes（也称为morph targets）来实现。VRM 规范定义了一套标准的表情blendshapes，但并不是所有的VRM模型都会包含所有这些表情。以下是VRM标准中定义的表情blendshapes及其对应的face-api.js表情：

neutral - 对应 face-api.js 的 "neutral"a - 张嘴（可用于 "surprised" 或 "happy"）e - 微笑（可用于 "happy"）i - 皱眉（可用于 "angry" 或 "disgusted"）o - 圆嘴（可用于 "surprised"）u - 撅嘴（可用于 "sad" 或 "disgusted"）blink - 眨眼blink_l - 左眼眨眼blink_r - 右眼眨眼joy - 开心（对应 "happy"）angry - 生气（对应 "angry"）sorrow - 悲伤（对应 "sad"）fun - 有趣（可用于 "happy" 或 "surprised"）lookUp - 向上看lookDown - 向下看lookLeft - 向左看lookRight - 向右看

回答

开发了一个网站ai聊天助手

一个月开发一套类似coze的智能体平台

部署一套内网离线ai助理

私有ai助理开发

类似如家的租房app开发

h5手机端考试网站开发

开发一个短剧解锁剧集的小程序

我要开发一个酒类拍卖交易平台

开发艺术品拍卖收藏买画卖画h5网站

帮我做个数字货币交易所网站

php如何检测mp3或wav音频时长？

ai如何在直播间介绍摄像头拍摄的实物智能实时与观众公屏互动？

ai如何理解长视频内容智能剪辑成精华短视频？

python如何实时在麦克风中识别多个不同人说话的声音并转成文字？

DeepSeek NSA技术是啥？

有哪些免费rpa可以自动抓取网页列表数据？

微信小程序如何保持屏幕不息屏常亮？

微信小程序麦克风录音时如何保持熄屏继续录音？

微信语音识别如何屏蔽手机扬声器播放的声音，只识别真人说话？

微信小程序如何只识别指定人说话声音转文字？