ScreenCaptureKit: AVAudioPCMBuffer Conversion Issue

[samueldurantes]

Hi folks,

I'm trying to generate auto-subtitles in real-time using ScreenCaptureKit API and SwiftWhisper, but whisper always returns no transcript to me.

I suspect it is the conversion of AVAudioPCMBuffer to Float[], but I have tried to convert in several ways but none were effective.

Does anyone know what the problem might be?

Here is my code:

import AVFoundation
import Combine
import Foundation
import OSLog
import ScreenCaptureKit
import SwiftUI

class Capturer: ObservableObject, @unchecked Sendable {
    private let logger = Logger()
    private let captureEngine = CaptureEngine()
    private let whisper = Whisper()
    private var audioBuffer: [Float] = []
    
    @Published var isRunning = false
    @Published var currentCaption = ""
    
    private let samplesPerProcess = 16000
    
    private var streamConfiguration: SCStreamConfiguration {
        let config = SCStreamConfiguration()
        config.capturesAudio = true
        config.minimumFrameInterval = CMTime(value: 1, timescale: 60)
        config.queueDepth = 5
        return config
    }
    
    private func processAudioBuffer(_ newSamples: [Float]) async {
        audioBuffer.append(contentsOf: newSamples)
        
        while audioBuffer.count >= samplesPerProcess {
            let chunk = Array(audioBuffer.prefix(samplesPerProcess))
            let text = await whisper.extractTextFromAudio(chunk)
            print("Whisper output: \(text)")
            audioBuffer.removeFirst(samplesPerProcess)
        }
    }

    func start() async {
        guard !isRunning else { return }
        
        do {
            let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false)
            guard let display = content.displays.first else { return }
            
            let excludedApps = content.applications.filter {
                Bundle.main.bundleIdentifier == $0.bundleIdentifier
            }
            
            let filter = SCContentFilter(
                display: display,
                excludingApplications: excludedApps,
                exceptingWindows: []
            )
            
            DispatchQueue.main.async {
                self.isRunning = true
            }
            
            for try await frames in captureEngine.startCapture(configuration: streamConfiguration, filter: filter) {
                await processAudioBuffer(frames as! [Float])
            }
            
        } catch {
            logger.error("\(error.localizedDescription)")
            DispatchQueue.main.async {
                self.isRunning = false
            }
        }
    }

    func stop() async {
        await captureEngine.stopCapture()
        DispatchQueue.main.async {
            self.isRunning = false
            self.currentCaption = ""
        }
    }
}

class CaptureEngine: NSObject, @unchecked Sendable {
    private let logger = Logger()
    private(set) var stream: SCStream?
    private var streamOutput: CaptureEngineStreamOutput?
    private var continuation: AsyncThrowingStream<Any, Error>.Continuation?

    func startCapture(configuration: SCStreamConfiguration, filter: SCContentFilter) -> AsyncThrowingStream<Any, Error> {
        AsyncThrowingStream { continuation in
            let streamOutput = CaptureEngineStreamOutput(continuation)
            self.streamOutput = streamOutput
            self.continuation = continuation

            do {
                stream = SCStream(filter: filter, configuration: configuration, delegate: streamOutput)
                try stream?.addStreamOutput(streamOutput, type: .screen, sampleHandlerQueue: .global())
                try stream?.addStreamOutput(streamOutput, type: .audio, sampleHandlerQueue: .global())
                stream?.startCapture()
            } catch {
                logger.error("Capture start error: \(error)")
            }
        }
    }

    func stopCapture() async {
        do {
            try await stream?.stopCapture()
            continuation?.finish()
        } catch {
            continuation?.finish(throwing: error)
        }
    }
}

private class CaptureEngineStreamOutput: NSObject, SCStreamOutput, SCStreamDelegate {
    private let logger = Logger()
    private var continuation: AsyncThrowingStream<Any, Error>.Continuation?
    private lazy var formatConverter: AVAudioConverter? = nil
    
    init(_ continuation: AsyncThrowingStream<Any, Error>.Continuation?) {
        self.continuation = continuation
    }

    func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of outputType: SCStreamOutputType) {
        guard sampleBuffer.isValid, outputType == .audio,
              let buffer = sampleBuffer.asPCMBuffer else { return }

        if formatConverter == nil {
            let outputFormat = AVAudioFormat(
                commonFormat: .pcmFormatFloat32,
                sampleRate: 16000,
                channels: 1,
                interleaved: false
            )!
            formatConverter = AVAudioConverter(from: buffer.format, to: outputFormat)
        }
        
        guard let converter = formatConverter else { return }
        let duration = Double(buffer.frameCapacity) / buffer.format.sampleRate
        let outputFrameCapacity = AVAudioFrameCount(16000 * duration)
        
        guard let outputBuffer = AVAudioPCMBuffer(
            pcmFormat: converter.outputFormat,
            frameCapacity: outputFrameCapacity
        ) else { return }
        
        var error: NSError?
        let status = converter.convert(
            to: outputBuffer,
            error: &error,
            withInputFrom: { _, outStatus in
                outStatus.pointee = .haveData
                return buffer
            }
        )
        
        if status == .error {
            print("Conversion error: \(error?.localizedDescription ?? "unknown")")
            return
        }

        if let samples = try? decodePCMBuffer(outputBuffer) {
            continuation?.yield(samples)
        }
    }

    func stream(_ stream: SCStream, didStopWithError error: Error) {
        logger.error("Stream stopped: \(error)")
        continuation?.finish(throwing: error)
    }
    
    func decodePCMBuffer(_ buffer: AVAudioPCMBuffer) throws -> [Float] {
        guard let floatChannelData = buffer.floatChannelData else {
            throw NSError(domain: "Invalid PCM Buffer", code: 0)
        }
        
        let channelCount = Int(buffer.format.channelCount)
        let frameLength = Int(buffer.frameLength)
        
        var floats: [Float] = Array(repeating: 0, count: frameLength * channelCount)
        
        for channel in 0..<channelCount {
            let channelData = floatChannelData[channel]
            for frame in 0..<frameLength {
                floats[frame * channelCount + channel] = channelData[frame]
            }
        }
        
        return floats
    }
}

extension CMSampleBuffer {
    var asPCMBuffer: AVAudioPCMBuffer? {
        try? withAudioBufferList { audioBufferList, _ in
            guard let desc = formatDescription?.audioStreamBasicDescription,
                  let format = AVAudioFormat(standardFormatWithSampleRate: desc.mSampleRate,
                                           channels: desc.mChannelsPerFrame) else { return nil }
            return AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: audioBufferList.unsafePointer)
        }
    }
}

[samueldurantes]

Any ideas? cc @jordibruin