Record audio in Wave apps

[mturoci]

This snippet shows how to access the user's microphone, record the audio and upload it to the Wave server for further processing. It has been tested against modern Chrome, Firefox, and Safari.

Demo

Screen.Recording.2022-06-24.at.3.52.25.PM.mov

Code for Wave app

The demo app consists of the simplest UI possible - single button that triggers browser Javascript. After the recording is stopped, the audio file is converted to wav format to have it unified across all browsers, uploaded to Wave server and downloaded to wave app for local consumption (e.g. feeding ML model).

from h2o_wave import main, app, Q, ui

@app('/')
async def serve(q: Q):
    if not q.client.initialized:
        q.page['form'] = ui.form_card(box='1 1 2 1', items=[ui.button(name='start_recording', label='Start recording')])
        # Load JS polyfills to be able to record on older browsers as well.
        q.page['meta'] = ui.meta_card(
            box='',
            scripts=[
              ui.script('https://cdn.jsdelivr.net/npm/opus-media-recorder@latest/OpusMediaRecorder.umd.js', asynchronous=False),
              ui.script('https://cdn.jsdelivr.net/npm/opus-media-recorder@latest/encoderWorker.umd.js', asynchronous=False),
            ]
        )
        # Load initial JS from file record_utils.js.
        with open('record.js', encoding='utf-8') as f:
            q.page['meta'].script = ui.inline_script(f.read())
        q.client.initialized = True
    if q.args.start_recording:
        # Start recording via JS.
        q.page['meta'].script = ui.inline_script('startRecording()')
        # Update Wave UI.
        btn = q.page['form'].items[0].button
        btn.name = 'stop_recording'
        btn.label = 'Stop recording'
    elif q.args.stop_recording:
        # Stop recording via JS.
        q.page['meta'].script = ui.inline_script('stopRecording()')
        # Update Wave UI.
        btn = q.page['form'].items[0].button
        btn.name = 'start_recording'
        btn.label = 'Start recording'

    if q.events.audio and q.events.audio.captured:
        # Download the result audio locally and consume it in a model.
        local_audio_file = await q.site.download(q.events.audio.captured, '.')
        # Just for demo purposes, allow for listening to recorded file.
        q.page['form'].items = [ui.text(f'Listen to [recording]({q.events.audio.captured})')]

    await q.page.save()

Javascript (JS)

Since the microphone is present in the browser, we have no other chance than use custom JS within Wave app.

// Shim for Safari.
window.AudioContext = window.AudioContext || window.webkitAudioContext

function audioBufferToWav(buffer, opt) {
  opt = opt || {}

  var numChannels = buffer.numberOfChannels
  var sampleRate = buffer.sampleRate
  var format = opt.float32 ? 3 : 1
  var bitDepth = format === 3 ? 32 : 16

  var result
  if (numChannels === 2) {
    result = interleave(buffer.getChannelData(0), buffer.getChannelData(1))
  } else {
    result = buffer.getChannelData(0)
  }

  return encodeWAV(result, format, sampleRate, numChannels, bitDepth)
}

function encodeWAV(samples, format, sampleRate, numChannels, bitDepth) {
  var bytesPerSample = bitDepth / 8
  var blockAlign = numChannels * bytesPerSample

  var buffer = new ArrayBuffer(44 + samples.length * bytesPerSample)
  var view = new DataView(buffer)

  /* RIFF identifier */
  writeString(view, 0, 'RIFF')
  /* RIFF chunk length */
  view.setUint32(4, 36 + samples.length * bytesPerSample, true)
  /* RIFF type */
  writeString(view, 8, 'WAVE')
  /* format chunk identifier */
  writeString(view, 12, 'fmt ')
  /* format chunk length */
  view.setUint32(16, 16, true)
  /* sample format (raw) */
  view.setUint16(20, format, true)
  /* channel count */
  view.setUint16(22, numChannels, true)
  /* sample rate */
  view.setUint32(24, sampleRate, true)
  /* byte rate (sample rate * block align) */
  view.setUint32(28, sampleRate * blockAlign, true)
  /* block align (channel count * bytes per sample) */
  view.setUint16(32, blockAlign, true)
  /* bits per sample */
  view.setUint16(34, bitDepth, true)
  /* data chunk identifier */
  writeString(view, 36, 'data')
  /* data chunk length */
  view.setUint32(40, samples.length * bytesPerSample, true)
  if (format === 1) { // Raw PCM
    floatTo16BitPCM(view, 44, samples)
  } else {
    writeFloat32(view, 44, samples)
  }

  return buffer
}

function interleave(inputL, inputR) {
  var length = inputL.length + inputR.length
  var result = new Float32Array(length)

  var index = 0
  var inputIndex = 0

  while (index < length) {
    result[index++] = inputL[inputIndex]
    result[index++] = inputR[inputIndex]
    inputIndex++
  }
  return result
}

function writeFloat32(output, offset, input) {
  for (var i = 0; i < input.length; i++, offset += 4) {
    output.setFloat32(offset, input[i], true)
  }
}

function floatTo16BitPCM(output, offset, input) {
  for (var i = 0; i < input.length; i++, offset += 2) {
    var s = Math.max(-1, Math.min(1, input[i]))
    output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true)
  }
}

function writeString(view, offset, string) {
  for (var i = 0; i < string.length; i++) {
    view.setUint8(offset + i, string.charCodeAt(i))
  }
}

// Safari does not support promise-based decodeAudioData, need to use callback instead.
const decodeAudioData = buffer => new Promise((res, rej) => {
  new AudioContext().decodeAudioData(buffer, res, rej)
})

const startRecording = async () => {
  const data = []
  // Ask for mic permissions.
  const stream = await navigator.mediaDevices.getUserMedia({ video: false, audio: true })
  window.stream = stream
  // Use polyfill for older browsers.
  if (!window.MediaRecorder) {
    window.MediaRecorder = OpusMediaRecorder
    window.recorder = new MediaRecorder(stream, {}, {
      OggOpusEncoderWasmPath: 'https://cdn.jsdelivr.net/npm/opus-media-recorder@latest/OggOpusEncoder.wasm',
      WebMOpusEncoderWasmPath: 'https://cdn.jsdelivr.net/npm/opus-media-recorder@latest/WebMOpusEncoder.wasm'
    })
  }
  else window.recorder = new MediaRecorder(stream)
  // Handle incoming data.
  window.recorder.ondataavailable = e => data.push(e.data)
  window.recorder.start()
  window.recorder.onerror = e => { throw e.error || new Error(e.name) }
  window.recorder.onstop = async (e) => {
    const blob = new Blob(data)
    const fetchedBlob = await fetch(URL.createObjectURL(blob))
    const arrayBuffer = await fetchedBlob.arrayBuffer()
    // Convert to wav format.
    const wav = audioBufferToWav(await decodeAudioData(arrayBuffer))
    const formData = new FormData()
    formData.append('files', new Blob([wav], { type: 'audio/wave' }), 'sound.wav')
    // Send the audio file to Wave server.
    const res = await fetch(wave.uploadURL, { method: 'POST', body: formData })
    const { files } = await res.json()
    // Emit event (q.events.audio.captured) with a URL of the audio file at Wave server.
    window.wave.emit('audio', 'captured', files[0])
  }
}

const stopRecording = () => {
  window.recorder.stop()
  window.stream.getTracks().forEach(track => track.stop())
}

The interesting part is uploading the data to the Wave server

const res = await fetch(wave.uploadURL, { method: 'POST', body: formData })

and sending Wave events.

window.wave.emit('audio', 'captured', files[0])

The rest is out of the scope of Wave itself as it's deep Javascript, but feel free to copy-paste it. If it doesn't work as advertised, let us know :)

cc @vopani @lo5 @mtanco