javascript - 通过 WebSocket 发送音频

Question

我正在实现 getUserMedia() 来录制音频消息。我想录制音频，然后使用 websocket 将其传递给另一个对等方。我怎样才能做到这一点？我在 SO 上进行了搜索，但只发现了一些有趣的问题，但没有任何东西可以指导我找到可行的解决方案。我正在为我的应用程序的 websocket 部分使用 Pusher API。 这是我正在测试的代码。

$(document).on("click", ".audio-chat",function(){
  console.log('clicked');
  var channel = $('input[name="channelName"]').val();
  navigator.mediaDevices.getUserMedia({
    audio: true
  })
  .then(function(stream){
    var mediaRecorder = new MediaRecorder(stream);
    mediaRecorder.start();
    console.log(mediaRecorder.state);
    console.log("recorder started");

    mediaRecorder.ondataavailable = function(e) {
      chunks.push(e.data);
      console.log(chunks);
    }
    setTimeout(function(){
      mediaRecorder.stop();
      console.log(mediaRecorder.state);
      console.log("recorder stopped");
      var blob = new Blob(chunks, {'type':'audio/webm; codecs=opus'});
      //console.log(blob);
      chunks = [];
      const audioUrl = window.URL.createObjectURL(blob);
      var data = {channel:channel,message:audioUrl,socketId:socketId}
      $.post('api/message.php', data);
    }, 10000);
  });
});

Answer 1

不要依赖 Blob/CreateObjectURL，而是尝试依赖 arrayBuffer 的二进制传输，如 https://github.com/Ivan-Feofanov/ws-audio-api 中所做的那样.

您还需要一个编码器和一个_resampler。

相关代码看

https://github.com/Ivan-Feofanov/ws-audio-api/blob/master/src/ws-audio-api.js

Streamer: function(config, socket) {
            navigator.getUserMedia = (navigator.getUserMedia ||
                navigator.webkitGetUserMedia ||
                navigator.mozGetUserMedia ||
                navigator.msGetUserMedia);

            this.config = {};
            this.config.codec = this.config.codec || defaultConfig.codec;
            this.config.server = this.config.server || defaultConfig.server;
            this.sampler = new Resampler(44100, this.config.codec.sampleRate, 1, this.config.codec.bufferSize);
            this.parentSocket = socket;
            this.encoder = new OpusEncoder(this.config.codec.sampleRate, this.config.codec.channels, this.config.codec.app, this.config.codec.frameDuration);
            var _this = this;
            this._makeStream = function(onError) {
                navigator.getUserMedia({ audio: true }, function(stream) {
                    _this.stream = stream;
                    _this.audioInput = audioContext.createMediaStreamSource(stream);
                    _this.gainNode = audioContext.createGain();
                    _this.recorder = audioContext.createScriptProcessor(_this.config.codec.bufferSize, 1, 1);
                    _this.recorder.onaudioprocess = function(e) {
                        var resampled = _this.sampler.resampler(e.inputBuffer.getChannelData(0));
                        var packets = _this.encoder.encode_float(resampled);
                        for (var i = 0; i < packets.length; i++) {
                            if (_this.socket.readyState == 1) _this.socket.send(packets[i]);
                        }
                    };
                    _this.audioInput.connect(_this.gainNode);
                    _this.gainNode.connect(_this.recorder);
                    _this.recorder.connect(audioContext.destination);
                }, onError || _this.onError);
            }
        }

串流开始

WSAudioAPI.Streamer.prototype.start = function(onError) {
        var _this = this;

        if (!this.parentSocket) {
            this.socket = new WebSocket('wss://' + this.config.server.host + ':' + this.config.server.port);
        } else {
            this.socket = this.parentSocket;
        }

        this.socket.binaryType = 'arraybuffer';

        if (this.socket.readyState == WebSocket.OPEN) {
            this._makeStream(onError);
        } else if (this.socket.readyState == WebSocket.CONNECTING) {
            var _onopen = this.socket.onopen;
            this.socket.onopen = function() {
                if (_onopen) {
                    _onopen();
                }
                _this._makeStream(onError);
            }
        } else {
            console.error('Socket is in CLOSED state');
        }

        var _onclose = this.socket.onclose;
        this.socket.onclose = function() {
            if (_onclose) {
                _onclose();
            }
            if (_this.audioInput) {
                _this.audioInput.disconnect();
                _this.audioInput = null;
            }
            if (_this.gainNode) {
                _this.gainNode.disconnect();
                _this.gainNode = null;
            }
            if (_this.recorder) {
                _this.recorder.disconnect();
                _this.recorder = null;
            }
            _this.stream.getTracks()[0].stop();
            console.log('Disconnected from server');
        };
    };

播放器

WSAudioAPI.Player.prototype.start = function() {
        var _this = this;

        this.audioQueue = {
            buffer: new Float32Array(0),

            write: function(newAudio) {
                var currentQLength = this.buffer.length;
                newAudio = _this.sampler.resampler(newAudio);
                var newBuffer = new Float32Array(currentQLength + newAudio.length);
                newBuffer.set(this.buffer, 0);
                newBuffer.set(newAudio, currentQLength);
                this.buffer = newBuffer;
            },

            read: function(nSamples) {
                var samplesToPlay = this.buffer.subarray(0, nSamples);
                this.buffer = this.buffer.subarray(nSamples, this.buffer.length);
                return samplesToPlay;
            },

            length: function() {
                return this.buffer.length;
            }
        };

        this.scriptNode = audioContext.createScriptProcessor(this.config.codec.bufferSize, 1, 1);
        this.scriptNode.onaudioprocess = function(e) {
            if (_this.audioQueue.length()) {
                e.outputBuffer.getChannelData(0).set(_this.audioQueue.read(_this.config.codec.bufferSize));
            } else {
                e.outputBuffer.getChannelData(0).set(_this.silence);
            }
        };
        this.gainNode = audioContext.createGain();
        this.scriptNode.connect(this.gainNode);
        this.gainNode.connect(audioContext.destination);

        if (!this.parentSocket) {
            this.socket = new WebSocket('wss://' + this.config.server.host + ':' + this.config.server.port);
        } else {
            this.socket = this.parentSocket;
        }
        //this.socket.onopen = function () {
        //    console.log('Connected to server ' + _this.config.server.host + ' as listener');
        //};
        var _onmessage = this.parentOnmessage = this.socket.onmessage;
        this.socket.onmessage = function(message) {
            if (_onmessage) {
                _onmessage(message);
            }
            if (message.data instanceof Blob) {
                var reader = new FileReader();
                reader.onload = function() {
                    _this.audioQueue.write(_this.decoder.decode_float(reader.result));
                };
                reader.readAsArrayBuffer(message.data);
            }
        };
        //this.socket.onclose = function () {
        //    console.log('Connection to server closed');
        //};
        //this.socket.onerror = function (err) {
        //    console.log('Getting audio data error:', err);
        //};
      };