WebRtc06: 音视频数据采集

音视频采集API

在这里插入图片描述通过getUserMedia这个API去获取视频音频，
通过constraints这个对象去配置偏好，比如视频宽高、音频降噪等

测试代码

index.html

<html><head><title>WebRtc capture video and audio</title></head><body><video autoplay playsinline id="player"></video><script src="./js/client.js"></script></body>
</html>

client.js

'use strict'var videoplay = document.querySelector('video#player');
function getMediaStream(stream) {videoplay.srcObject = stream;
}
function handleError(err) {console.log('getUserMedia error: ', err);
}if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {console.log('getUserMedia is not supported');
} else {var constraints = {video : true,audio : true}navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).catch(handleError);
}

会打开前置摄像头并且输出视频音频

浏览器适配

getUserMedia适配

getUserMedia (www)
webkitGetUserMedia(google)
mozGetUserMedia(firefox)
每个厂商的API名称都不一样
比如在前面的index.html中加上下述代码

<script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>

获取音视频设备的访问权限

在前面代码的基础上，返回navigator.mediaDevices.enumerateDevices()，从而得到每一个媒体信息
然后通过添加到HTML的选择器中实现打印不同设备信息
index.html

<html><head><title>WebRtc capture video and audio</title></head><body><div><label>audioSource:</label><select id="audioSource"></select></div><div><label>audioOutput:</label><select id="audioOutput"></select></div><div><label>videoSource:</label><select id="videoSource"></select></div><video autoplay playsinline id="player"></video><script src="https://webrtc.github.io/adapter/adapter-latest.js"></script><script src="./js/client.js"></script></body>
</html>

client.js

'use strict'var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');var videoplay = document.querySelector('video#player');function getDevice(deviceInfos) {deviceInfos.forEach(function(deviceInfos) {var option = document.createElement('option');option.text = deviceInfos.label;option.value = deviceInfos.deviceId;if (deviceInfos.kind == 'audioinput') {audioSource.appendChild(option);} else if (deviceInfos.kind == 'audiooutput') {audioOutput.appendChild(option);} else if (deviceInfos.kind == 'videoinput') {videoSource.appendChild(option);}});
}function getMediaStream(stream) {videoplay.srcObject = stream;return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {console.log('getUserMedia error: ', err);
}if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {console.log('getUserMedia is not supported');
} else {var constraints = {video : true,audio : true}navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
}

音视频采集约束

视频约束

width
height
aspectRatio 宽高比
frameRate 帧率
facingMode
- user 前置摄像头
- environment 后置摄像头
- left 前置左侧摄像头
- right 前置右侧摄像头
resizeMode

设置constraints中视频的参数如下

var constraints = {video : {width: 1920,height: 1080,frameRate: 30,facingMode: 'environment'},audio : true}

音频约束

volume 音量 [0-1.0]
sampleRate 采样率
sampleSize 采样大小位深一般16位
echoCancellation 回音消除 true/false
autoGainControl 自动增益 true/false 在原有声音的基础上，增加音量
noiseSuppression 降噪 true/false
latency 延迟大小
channelCount 声道数量
deviceID
groupID 组ID

var constraints = {video : {width: 1920,height: 1080,frameRate: 30,facingMode: 'environment'},audio : {noiseSuppression: true,echoCancellation: true}}

整体例子：切换不同摄像头

'use strict'var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');var videoplay = document.querySelector('video#player');function getDevice(deviceInfos) {deviceInfos.forEach(function(deviceInfos) {var option = document.createElement('option');option.text = deviceInfos.label;option.value = deviceInfos.deviceId;console.log('lai: deviceInfo: ', deviceInfos.label);if (deviceInfos.kind == 'audioinput') {audioSource.appendChild(option);} else if (deviceInfos.kind == 'audiooutput') {audioOutput.appendChild(option);} else if (deviceInfos.kind == 'videoinput') {videoSource.appendChild(option);}});
}function getMediaStream(stream) {videoplay.srcObject = stream;return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {console.log('getUserMedia error: ', err);
}function start() {if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {console.log('getUserMedia is not supported');} else {var deviceId = videoSource.value;var constraints = {video : {width: 1920,height: 1080,frameRate: 30,deviceId : deviceId ? deviceId : undefined},audio : {noiseSuppression: true,echoCancellation: true}}navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);}
}
start();videoSource.onchange = start;

实战

视频渲染特效

浏览器视频特效

CSS filter: -webkit-filter/filter
不同浏览器的名称不一样，Chrome使用的是-webkit-filter，IE用的是filter

需要知道如何将video和filter关联

特效的底层调用都是OpenGL/Metal等等

常用特效

在这里插入图片描述

代码测试

在index.html中添加css和filter的selector，然后在js中指定播放器用的类
index.html

<html><head><title>WebRtc capture video and audio</title><style>.none {-webkit-filter: none;}.blur {-webkit-filter: blur(3px);}.grayscale {-webkit-filter:grayscale(1);}.invert {-webkit-filter: invert(1);}.sepia {-webkit-filter:sepia(1);}</style></head><body><div><label>audioSource:</label><select id="audioSource"></select></div><div><label>audioOutput:</label><select id="audioOutput"></select></div><div><label>videoSource:</label><select id="videoSource"></select></div><div><label>filter:</label><select id="filter"><option value="none"> None</option><option value="blur"> blur</option><option value="grayscale"> grayscale</option><option value="invert"> invert</option><option value="sepia"> sepia</option></select></div><video autoplay playsinline id="player"></video><script src="https://webrtc.github.io/adapter/adapter-latest.js"></script><script src="./js/client.js"></script></body>
</html>

client.js

'use strict'var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');var filterSelect = document.querySelector('select#filter');var videoplay = document.querySelector('video#player');function getDevice(deviceInfos) {deviceInfos.forEach(function(deviceInfos) {var option = document.createElement('option');option.text = deviceInfos.label;option.value = deviceInfos.deviceId;console.log('lai: deviceInfo: ', deviceInfos.label);if (deviceInfos.kind == 'audioinput') {audioSource.appendChild(option);} else if (deviceInfos.kind == 'audiooutput') {audioOutput.appendChild(option);} else if (deviceInfos.kind == 'videoinput') {videoSource.appendChild(option);}});
}function getMediaStream(stream) {videoplay.srcObject = stream;return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {console.log('getUserMedia error: ', err);
}function start() {if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {console.log('getUserMedia is not supported');} else {var deviceId = videoSource.value;var constraints = {video : {width: 1920,height: 1080,frameRate: 30,deviceId : deviceId ? deviceId : undefined},audio : {noiseSuppression: true,echoCancellation: true}}navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);}
}
start();videoSource.onchange = start;filterSelect.onchange = function() {videoplay.className = filterSelect.value;
}

从视频中获取图片

index.html

<html><head><title>WebRtc capture video and audio</title><style>.none {-webkit-filter: none;}.blur {-webkit-filter: blur(3px);}.grayscale {-webkit-filter:grayscale(1);}.invert {-webkit-filter: invert(1);}.sepia {-webkit-filter:sepia(1);}</style></head><body><div><label>audioSource:</label><select id="audioSource"></select></div><div><label>audioOutput:</label><select id="audioOutput"></select></div><div><label>videoSource:</label><select id="videoSource"></select></div><div><label>filter:</label><select id="filter"><option value="none"> None</option><option value="blur"> blur</option><option value="grayscale"> grayscale</option><option value="invert"> invert</option><option value="sepia"> sepia</option></select></div><video autoplay playsinline id="player"></video><div><button id="snapshot">Take snapshot</button></div><div><canvas id="picture"></canvas></div><script src="https://webrtc.github.io/adapter/adapter-latest.js"></script><script src="./js/client.js"></script></body>
</html>

client.js

'use strict'// devices
var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');// filter
var filterSelect = document.querySelector('select#filter');// picture
var snapshot = document.querySelector('button#snapshot');
var picture = document.querySelector('canvas#picture');
picture.width = 320;
picture.height = 240;var videoplay = document.querySelector('video#player');function getDevice(deviceInfos) {deviceInfos.forEach(function(deviceInfos) {var option = document.createElement('option');option.text = deviceInfos.label;option.value = deviceInfos.deviceId;if (deviceInfos.kind == 'audioinput') {audioSource.appendChild(option);} else if (deviceInfos.kind == 'audiooutput') {audioOutput.appendChild(option);} else if (deviceInfos.kind == 'videoinput') {videoSource.appendChild(option);}});
}function getMediaStream(stream) {videoplay.srcObject = stream;return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {console.log('getUserMedia error: ', err);
}function start() {if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {console.log('getUserMedia is not supported');} else {var deviceId = videoSource.value;var constraints = {video : {width: 1920,height: 1080,frameRate: 30,deviceId : deviceId ? deviceId : undefined},audio : {noiseSuppression: true,echoCancellation: true}}navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);}
}
start();videoSource.onchange = start;filterSelect.onchange = function() {videoplay.className = filterSelect.value;
}// 点击按钮进行截图，但是这里的截图保存到本地是没有滤镜效果的
snapshot.onclick = function() {picture.className = filterSelect.value;picture.getContext('2d').drawImage(videoplay, 0, 0, picture.width, picture.height);
}

只采集音频数据

临时屏蔽vidoplayer，设置constraints中的video为false，获取到的媒体流赋值给audioplayer

index.html

<html><head><title>WebRtc capture video and audio</title><style>.none {-webkit-filter: none;}.blur {-webkit-filter: blur(3px);}.grayscale {-webkit-filter:grayscale(1);}.invert {-webkit-filter: invert(1);}.sepia {-webkit-filter:sepia(1);}</style></head><body><div><label>audioSource:</label><select id="audioSource"></select></div><div><label>audioOutput:</label><select id="audioOutput"></select></div><div><label>videoSource:</label><select id="videoSource"></select></div><div><label>filter:</label><select id="filter"><option value="none"> None</option><option value="blur"> blur</option><option value="grayscale"> grayscale</option><option value="invert"> invert</option><option value="sepia"> sepia</option></select></div><!-- controls用于显示控制按钮 --><audio autoplay controls id="audioplayer" ></audio><!-- <video autoplay playsinline id="player"></video> --><div><button id="snapshot">Take snapshot</button></div><div><canvas id="picture"></canvas></div><script src="https://webrtc.github.io/adapter/adapter-latest.js"></script><script src="./js/client.js"></script></body>
</html>

client.js

'use strict'// devices
var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');// filter
var filterSelect = document.querySelector('select#filter');// picture
var snapshot = document.querySelector('button#snapshot');
var picture = document.querySelector('canvas#picture');
picture.width = 320;
picture.height = 240;// var videoplay = document.querySelector('video#player');
// 获取audioplayer
var audioplay = document.querySelector('audio#audioplayer');function getDevice(deviceInfos) {deviceInfos.forEach(function(deviceInfos) {var option = document.createElement('option');option.text = deviceInfos.label;option.value = deviceInfos.deviceId;if (deviceInfos.kind == 'audioinput') {audioSource.appendChild(option);} else if (deviceInfos.kind == 'audiooutput') {audioOutput.appendChild(option);} else if (deviceInfos.kind == 'videoinput') {videoSource.appendChild(option);}});
}function getMediaStream(stream) {// videoplay.srcObject = stream;audioplay.srcObject = stream;return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {console.log('getUserMedia error: ', err);
}function start() {if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {console.log('getUserMedia is not supported');} else {var deviceId = videoSource.value;var constraints = {// video : {//     width: 1920,//     height: 1080,//     frameRate: 30,//     deviceId : deviceId ? deviceId : undefined// },video : false,audio : true}navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);}
}
start();videoSource.onchange = start;filterSelect.onchange = function() {videoplay.className = filterSelect.value;
}snapshot.onclick = function() {picture.className = filterSelect.value;picture.getContext('2d').drawImage(videoplay, 0, 0, picture.width, picture.height);
}

MediaStreamAPI及获取视频约束

MediaStream API

MediaStream.addTrack()
MediaStream.removeTrack()
MediaStream.getVideoTracks()
MediaStream.getAudioTracks()

MediaStream事件

MediaStream.onaddtrack()
MediaStream.onremovetrack()
MediaStream.onended()

获取视频约束

index.html

<html><head><title>WebRtc capture video and audio</title><style>.none {-webkit-filter: none;}.blur {-webkit-filter: blur(3px);}.grayscale {-webkit-filter:grayscale(1);}.invert {-webkit-filter: invert(1);}.sepia {-webkit-filter:sepia(1);}</style></head><body><div><label>audioSource:</label><select id="audioSource"></select></div><div><label>audioOutput:</label><select id="audioOutput"></select></div><div><label>videoSource:</label><select id="videoSource"></select></div><div><label>filter:</label><select id="filter"><option value="none"> None</option><option value="blur"> blur</option><option value="grayscale"> grayscale</option><option value="invert"> invert</option><option value="sepia"> sepia</option></select></div><!-- controls用于显示控制按钮 --><!-- <audio autoplay controls id="audioplayer" ></audio> --><table><tr><td><video autoplay playsinline id="player"></video></td><!-- 显示视频约束 --><td><div id = 'constraints' class='output'></div></td></tr></table><div><button id="snapshot">Take snapshot</button></div><div><canvas id="picture"></canvas></div><script src="https://webrtc.github.io/adapter/adapter-latest.js"></script><script src="./js/client.js"></script></body>
</html>

client.js

'use strict'// devices
var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');// filter
var filterSelect = document.querySelector('select#filter');// picture
var snapshot = document.querySelector('button#snapshot');
var picture = document.querySelector('canvas#picture');
picture.width = 320;
picture.height = 240;var videoplay = document.querySelector('video#player');
// 获取audioplayer
// var audioplay = document.querySelector('audio#audioplayer');var divConstraints = document.querySelector('div#constraints');function getDevice(deviceInfos) {deviceInfos.forEach(function(deviceInfos) {var option = document.createElement('option');option.text = deviceInfos.label;option.value = deviceInfos.deviceId;if (deviceInfos.kind == 'audioinput') {audioSource.appendChild(option);} else if (deviceInfos.kind == 'audiooutput') {audioOutput.appendChild(option);} else if (deviceInfos.kind == 'videoinput') {videoSource.appendChild(option);}});
}function getMediaStream(stream) {videoplay.srcObject = stream;var videoTrack = stream.getVideoTracks()[0];var videoConstraints = videoTrack.getSettings();divConstraints.textContent = JSON.stringify(videoConstraints, null, 2);// audioplay.srcObject = stream;return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {console.log('getUserMedia error: ', err);
}function start() {if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {console.log('getUserMedia is not supported');} else {var deviceId = videoSource.value;var constraints = {video : {width: 640,height: 480,frameRate: 30,deviceId : deviceId ? deviceId : undefined},// video : false,audio : true}navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);}
}
start();videoSource.onchange = start;filterSelect.onchange = function() {videoplay.className = filterSelect.value;
}snapshot.onclick = function() {picture.className = filterSelect.value;picture.getContext('2d').drawImage(videoplay, 0, 0, picture.width, picture.height);
}