I am using javascript sample to receive real-time microphone audio and transmit it as an audio stream to a WebSocket server. I am using the following simple Python code as a WebSocket server to receive the stream and play it using PyAudio. However, I am getting only noise when playing the audio. I tested using pyaudio functionality directly with the microphone, and it works fine. But when using JavaScript, there seems to be an issue. Is there a problem with PyAudio reading this streaming format?
python webssocket server
import pyaudio
import numpy as np
import asyncio
import websockets
CHUNK_SIZE = 1024
audio = pyaudio.PyAudio()
stream = audio.open(format=pyaudio.paInt16,
channels=1,
rate=16000,
output=True,
frames_per_buffer=CHUNK_SIZE)
async def websocket_handler(websocket, path):
try:
async for message in websocket:
print(message)
audio_data = np.frombuffer(message, dtype=np.int16)
stream.write(audio_data.tobytes())
#stream.write(message)
finally:
stream.stop_stream()
stream.close()
audio.terminate()
async def start_websocket_server():
async with websockets.serve(websocket_handler, '', 8888):
await asyncio.Future() # Keep the server running
if __name__ == '__main__':
asyncio.run(start_websocket_server())
python webssocket client
import pyaudio
import asyncio
import websockets
def record_microphone(stream):
CHUNK = 1024
while True:
data = stream.read(CHUNK)
yield data
async def send_audio():
async with websockets.connect('ws://xxx.xxx.x.xxx:8888') as ws:
p = pyaudio.PyAudio()
# obtain the index of available mic
mic_device_index = None
for i in range(p.get_device_count()):
device_info = p.get_device_info_by_index(i)
if device_info['maxInputChannels'] > 0:
mic_device_index = i
break
if mic_device_index is None:
print("there is no mic")
return
stream = p.open(format=pyaudio.paInt16,
channels=1,
rate=44100,
input=True,
frames_per_buffer=1024,
input_device_index=mic_device_index)
for data in record_microphone(stream):
await ws.send(data)
asyncio.get_event_loop().run_until_complete(send_audio())
javascript sample
//================= CONFIG =================
// Global Variables
let websocket_uri = 'ws://127.0.0.1:9001';
let bufferSize = 4096,
AudioContext,
context,
processor,
input,
globalStream,
websocket;
// Initialize WebSocket
initWebSocket();
//================= RECORDING =================
function startRecording() {
streamStreaming = true;
AudioContext = window.AudioContext || window.webkitAudioContext;
context = new AudioContext({
// if Non-interactive, use 'playback' or 'balanced' // https://developer.mozilla.org/en-US/docs/Web/API/AudioContextLatencyCategory
latencyHint: 'interactive',
});
processor = context.createScriptProcessor(bufferSize, 1, 1);
processor.connect(context.destination);
context.resume();
var handleSuccess = function (stream) {
globalStream = stream;
input = context.createMediaStreamSource(stream);
input.connect(processor);
processor.onaudioprocess = function (e) {
var left = e.inputBuffer.getChannelData(0);
var left16 = downsampleBuffer(left, 44100, 16000);
websocket.send(left16);
};
};
navigator.mediaDevices.getUserMedia({audio: true, video: false}).then(handleSuccess);
} // closes function startRecording()
function stopRecording() {
streamStreaming = false;
let track = globalStream.getTracks()[0];
track.stop();
input.disconnect(processor);
processor.disconnect(context.destination);
context.close().then(function () {
input = null;
processor = null;
context = null;
AudioContext = null;
});
} // closes function stopRecording()
function initWebSocket() {
// Create WebSocket
websocket = new WebSocket(websocket_uri);
//console.log("Websocket created...");
// WebSocket Definitions: executed when triggered webSocketStatus
websocket.onopen = function() {
console.log("connected to server");
//websocket.send("CONNECTED TO YOU");
document.getElementById("webSocketStatus").innerHTML = 'Connected';
}
websocket.onclose = function(e) {
console.log("connection closed (" + e.code + ")");
document.getElementById("webSocketStatus").innerHTML = 'Not Connected';
}
websocket.onmessage = function(e) {
//console.log("message received: " + e.data);
console.log(e.data);
try {
result = JSON.parse(e.data);
} catch (e) {
$('.message').html('Error retrieving data: ' + e);
}
if (typeof(result) !== 'undefined' && typeof(result.error) !== 'undefined') {
$('.message').html('Error: ' + result.error);
}
else {
$('.message').html('Welcome!');
}
}
} // closes function initWebSocket()
function downsampleBuffer (buffer, sampleRate, outSampleRate) {
if (outSampleRate == sampleRate) {
return buffer;
}
if (outSampleRate > sampleRate) {
throw 'downsampling rate show be smaller than original sample rate';
}
var sampleRateRatio = sampleRate / outSampleRate;
var newLength = Math.round(buffer.length / sampleRateRatio);
var result = new Int16Array(newLength);
var offsetResult = 0;
var offsetBuffer = 0;
while (offsetResult < result.length) {
var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
var accum = 0,
count = 0;
for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
accum += buffer[i];
count++;
}
result[offsetResult] = Math.min(1, accum / count) * 0x7fff;
offsetResult++;
offsetBuffer = nextOffsetBuffer;
}
return result.buffer;
} // closes function downsampleBuffer()