Connection of varispeed with RemoteIO in iOS

Question

I am working with audio units to play and change the speed of playback. Since AudioGraph is deprecated.

What I have done, I have successfully played Audio coming from UDP via audio-units and made connections like:

converterUnit -> varispeed -> outConverterUnit -> RemoteIO (Out)

Our format for playing is int16(PCM), but varispeed requires float datatype, So we are using converters for varispeed.

Here is my code:

var ioFormat = CAStreamBasicDescription(
    sampleRate: 48000.0,
    numChannels: 1,
    pcmf: .int16,
    isInterleaved: false
    )

var varispeedFormat = CAStreamBasicDescription(
    sampleRate: 16000,
    numChannels: 1,
    pcmf: .float32,
    isInterleaved: false
)


init(_ client: UDPClient, _ tcpClient: TCPClient, _ opusHelper: OpusHelper, _ tvTemp: UILabel) {
    super.init()

let success = initCircularBuffer(&circularBuffer, 4096)
if success {
    print("Circular buffer init was successful")
} else {
    print("Circular buffer init not successful")
}

self.opusHelper = opusHelper

self.tvTemp = tvTemp
monotonicTimer = MonotonicTimer()


self.udpClient = client
self.tcpClient = tcpClient

//Creating Description for REMOTE IO
var outputDesc = AudioComponentDescription(
    componentType: OSType(kAudioUnitType_Output),
    componentSubType: OSType(kAudioUnitSubType_VoiceProcessingIO),
    componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
    componentFlags: 0,
    componentFlagsMask: 0
)

let inputComponent = AudioComponentFindNext(nil, &outputDesc)

check(error: AudioComponentInstanceNew(inputComponent!, &outputUnit), description: "Output unit instance new failed")

//Creating Description for converterUnit
var firstConverterDesc = AudioComponentDescription(
    componentType: OSType(kAudioUnitType_FormatConverter),
    componentSubType: OSType(kAudioUnitSubType_AUConverter),
    componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
    componentFlags: 0,
    componentFlagsMask: 0
)

let firstConverterComponent = AudioComponentFindNext(nil, &firstConverterDesc)
check(error: AudioComponentInstanceNew(firstConverterComponent!, &firstConverterUnit), description: "First converter unit instance new failed")

//Creating Description for Varispeed Unit
var variSpeedConverterDesc = AudioComponentDescription(
    componentType: OSType(kAudioUnitType_FormatConverter),
    componentSubType: OSType(kAudioUnitSubType_Varispeed),
    componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
    componentFlags: 0,
    componentFlagsMask: 0
)

let variSpeedConverterComponent = AudioComponentFindNext(nil, &variSpeedConverterDesc)
check(error: AudioComponentInstanceNew(variSpeedConverterComponent!, &varispeedUnit), description: "First converter unit instance new failed")


//Creating Description for outConverter
var secondConverterDesc = AudioComponentDescription(
    componentType: OSType(kAudioUnitType_FormatConverter),
    componentSubType: OSType(kAudioUnitSubType_AUConverter),
    componentManufacturer: OSType(kAudioUnitManufacturer_Apple),
    componentFlags: 0,
    componentFlagsMask: 0
)

let secondConverterComponent = AudioComponentFindNext(nil, &secondConverterDesc)
check(error: AudioComponentInstanceNew(secondConverterComponent!, &secondConverterUnit), description: "Second converter unit instance new failed")


//Converting incoming bytes to AUConverter (Float 32 format)
check(error: AudioUnitSetProperty(
    firstConverterUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
    AudioUnitScope(kAudioUnitScope_Output),
    0,
    &varispeedFormat,
    MemoryLayoutStride.SizeOf32(varispeedFormat)
    ),
      description: "Failed to set input of second converter to our temp format"
)

//Putting converted bytes in varispeed unit
check(error: AudioUnitSetProperty(
    varispeedUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
    AudioUnitScope(kAudioUnitScope_Input),
    0,
    &varispeedFormat,
    MemoryLayoutStride.SizeOf32(varispeedFormat)
    ),
      description: "Failed to set input format as varispeed format of the second converter unit"
)

//Getting converted bytes from varispeed unit
check(error: AudioUnitSetProperty(
    varispeedUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
    AudioUnitScope(kAudioUnitScope_Output),
    0,
    &varispeedFormat,
    MemoryLayoutStride.SizeOf32(varispeedFormat)
    ),
      description: "Failed to set input format as varispeed format of the second converter unit"
)

//Putting converted bytes in outConverterUnit
check(error: AudioUnitSetProperty(
    secondConverterUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
    AudioUnitScope(kAudioUnitScope_Input),
    0,
    &varispeedFormat,
    MemoryLayoutStride.SizeOf32(varispeedFormat)
    ),
      description: "Failed to set input of second converter to our temp11 format"
)

//Getting converted bytes from outConverterUnit in int16(PCM)
check(error: AudioUnitSetProperty(
    secondConverterUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_StreamFormat),
    AudioUnitScope(kAudioUnitScope_Output),
    0,
    &ioFormat,
    MemoryLayoutStride.SizeOf32(ioFormat)
    ),
      description: "Failed to set input of second converter to our temp11 format"
)


//Connecting firstConverter to varispeed
var tempConnection = AudioUnitConnection(
    sourceAudioUnit: firstConverterUnit!,
    sourceOutputNumber: 0,
    destInputNumber: 0
)

check(error: AudioUnitSetProperty(
    varispeedUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_MakeConnection),
    AudioUnitScope(kAudioUnitScope_Input),
    0,
    &tempConnection,
    MemoryLayoutStride.SizeOf32(AudioUnitConnection())
    ),
      description: "Failed to connect second converter to output Unit"
)

//Connecting verispeedUnit to to outConverter
var temp1Connection = AudioUnitConnection(
    sourceAudioUnit: varispeedUnit!,
    sourceOutputNumber: 0,
    destInputNumber: 0
)

check(error: AudioUnitSetProperty(
    secondConverterUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_MakeConnection),
    AudioUnitScope(kAudioUnitScope_Input),
    0,
    &temp1Connection,
    MemoryLayoutStride.SizeOf32(AudioUnitConnection())
    ),
      description: "Failed to connect second converter to output Unit"
)

//Connecting outConverter to outputUnit
var secondToOutputConnection = AudioUnitConnection(
    sourceAudioUnit: secondConverterUnit!,
    sourceOutputNumber: 0,
    destInputNumber: 0
)

check(error: AudioUnitSetProperty(
    outputUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_MakeConnection),
    AudioUnitScope(kAudioUnitScope_Input),
    0,
    &secondToOutputConnection,
    MemoryLayoutStride.SizeOf32(AudioUnitConnection())
    ),
      description: "Failed to connect second converter to output Unit"
)

check(error: AudioUnitInitialize(outputUnit!), description: "Failed to init output unit")
check(error: AudioUnitInitialize(firstConverterUnit!), description: "Failed to init first converter unit")
check(error: AudioUnitInitialize(varispeedUnit!), description: "Failed to init varispeed unit")
check(error: AudioUnitInitialize(secondConverterUnit!), description: "Failed to init second converter unit")


var playbackCallback = AURenderCallbackStruct(
    inputProc: AudioController_PlaybackCallback,
    inputProcRefCon: UnsafeMutableRawPointer(Unmanaged.passUnretained(self).toOpaque())
)

check(error: AudioUnitSetProperty(
    outputUnit!,
    AudioUnitPropertyID(kAudioUnitProperty_SetRenderCallback),
    AudioUnitScope(kAudioUnitScope_Input),
    kOutputBus,
    &playbackCallback,
    MemoryLayout<AURenderCallbackStruct>.size.ui
    ),
      description: "Failed to set recording render callback"
)
}

The parameter for changing the playback speed is given here.

func performPlayback(
            _ ioActionFlags: UnsafeMutablePointer<AudioUnitRenderActionFlags>,
            inTimeStamp: UnsafePointer<AudioTimeStamp>,
            inBufNumber: UInt32,
            inNumberFrames: UInt32,
            ioData: UnsafeMutablePointer<AudioBufferList>
        ) -> OSStatus {

let buffer = ioData[0].mBuffers

let bytesToCopy = ioData[0].mBuffers.mDataByteSize
var bufferTail: UnsafeMutableRawPointer?
//        print("BYTES TO COPY: \(bytesToCopy)")
self.availableBytes = 0
bufferTail = TPCircularBufferTail(&self.circularBuffer, &self.availableBytes)
bytesToWrite = min(bytesToCopy, self.availableBytes)

check(error: AudioUnitSetParameter(
       varispeedUnit!,
       AudioUnitParameterID(kVarispeedParam_PlaybackRate),
       AudioUnitScope(kAudioUnitScope_Global),
       0,
       AudioUnitParameterValue(1.9999000082426955),
       0
    ),
      description: "Failed to set parameter rate for varispeed unit"
)


print("BYTES TO WRITE: \(bytesToWrite)")
if bytesToWrite >= 3840 {
    memcpy(buffer.mData, bufferTail, Int(bytesToWrite))
    TPCircularBufferConsume(&self.circularBuffer, bytesToWrite)
} else {
    let silence = [Int16](repeating: 0, count: Int(bytesToCopy))
    memcpy(buffer.mData, silence, Int(bytesToCopy))
}
return noErr
}

The problem is, I don't feel any difference in voice if I use varispeed or not. Can anyone point out the problem in my code?

I have studied these answers and tried to implement them in our situation with no result. I think scopes and elements might be the problem.

@hotpaw2 can your confirm if the written code is connecting the AudioUnits together properly or not? — Muhammad Faizan, Mar 15 '20 at 09:24
@hotpaw2 should we connect another audio unit after the output scope/output bus of the output unit or before that? And can you please confirm that the render callback should be on the output unit or the effect audio unit? Thankyou for your help so far — shayan, Mar 15 '20 at 14:04
This is the comment section. Ask questions by asking a new question. One question per question here is usually better at getting answers. — hotpaw2, Mar 15 '20 at 14:09
@hotpaw2 , I'm trying to reach you from linkedIn and Twitter but couldn't get any response from your side. First thing, kindly give me your email address or any contact channel so that I can be able to share my problem with you directly or you can either start stack overflow messages thread ? — Muhammad Usman Bashir, Mar 15 '20 at 14:17

Connection of varispeed with RemoteIO in iOS

0 Answers0

Linked