Skip to content

🎙️ Add Real-time Buffer Monitoring and Voice Activity Detection (VAD) based on the Core Audio taps #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rongweiji wants to merge 5 commits into
base: main
Choose a base branch
from
Open

🎙️ Add Real-time Buffer Monitoring and Voice Activity Detection (VAD) based on the Core Audio taps #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7b237e3
Add real-time audio monitoring and voice activity detection features
rongweiji Jun 23, 2025
d511557
Update README.md
rongweiji Jun 23, 2025
0a92db6
clean the folder
rongweiji Jun 23, 2025
11c163e
Update README.md
rongweiji Jun 23, 2025
60765d8
fix the reference error for the additional files
rongweiji Jul 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 47 additions & 23 deletions AudioCap.xcodeproj/project.pbxproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,34 +7,40 @@
objects = {

/* Begin PBXBuildFile section */
CC537FC52E09E7CA00503A96 /* RealtimeAudioMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC537FC42E09E7CA00503A96 /* RealtimeAudioMonitor.swift */; };
CC537FC72E09E7D500503A96 /* VoiceActivityDetector.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC537FC62E09E7D500503A96 /* VoiceActivityDetector.swift */; };
CC98B1A62E17370900A4EDAB /* ProcessSelectionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC98B1A02E17370900A4EDAB /* ProcessSelectionView.swift */; };
CC98B1A72E17370900A4EDAB /* RootView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC98B1A42E17370900A4EDAB /* RootView.swift */; };
CC98B1A82E17370900A4EDAB /* RecordingIndicator.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC98B1A22E17370900A4EDAB /* RecordingIndicator.swift */; };
CC98B1A92E17370900A4EDAB /* FileProxyView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC98B19F2E17370900A4EDAB /* FileProxyView.swift */; };
CC98B1AA2E17370900A4EDAB /* RecordingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC98B1A32E17370900A4EDAB /* RecordingView.swift */; };
CC98B1AB2E17370900A4EDAB /* RealtimeVADView.swift in Sources */ = {isa = PBXBuildFile; fileRef = CC98B1A12E17370900A4EDAB /* RealtimeVADView.swift */; };
F431728D2BF68C0C00D918A3 /* AudioRecordingPermission.swift in Sources */ = {isa = PBXBuildFile; fileRef = F431728C2BF68C0C00D918A3 /* AudioRecordingPermission.swift */; };
F43172902BF6A92900D918A3 /* AudioProcessController.swift in Sources */ = {isa = PBXBuildFile; fileRef = F431728F2BF6A92900D918A3 /* AudioProcessController.swift */; };
F43172922BF6ABB000D918A3 /* ProcessSelectionView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F43172912BF6ABB000D918A3 /* ProcessSelectionView.swift */; };
F43172942BF6B01000D918A3 /* ProcessTap.swift in Sources */ = {isa = PBXBuildFile; fileRef = F43172932BF6B01000D918A3 /* ProcessTap.swift */; };
F43172962BF787C300D918A3 /* CoreAudioUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = F43172952BF787C300D918A3 /* CoreAudioUtils.swift */; };
F431729A2BF7A51A00D918A3 /* RecordingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F43172992BF7A51A00D918A3 /* RecordingView.swift */; };
F431729C2BF7B66A00D918A3 /* FileProxyView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F431729B2BF7B66A00D918A3 /* FileProxyView.swift */; };
F43172A12BF7BF1300D918A3 /* RecordingIndicator.swift in Sources */ = {isa = PBXBuildFile; fileRef = F43172A02BF7BF1300D918A3 /* RecordingIndicator.swift */; };
F47AD9422BF5B61E005B75AC /* AudioCapApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = F47AD9412BF5B61E005B75AC /* AudioCapApp.swift */; };
F47AD9442BF5B61E005B75AC /* RootView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F47AD9432BF5B61E005B75AC /* RootView.swift */; };
F47AD9462BF5B61F005B75AC /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = F47AD9452BF5B61F005B75AC /* Assets.xcassets */; };
F47AD9492BF5B61F005B75AC /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = F47AD9482BF5B61F005B75AC /* Preview Assets.xcassets */; };
/* End PBXBuildFile section */

/* Begin PBXFileReference section */
CC537FC42E09E7CA00503A96 /* RealtimeAudioMonitor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealtimeAudioMonitor.swift; sourceTree = "<group>"; };
CC537FC62E09E7D500503A96 /* VoiceActivityDetector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceActivityDetector.swift; sourceTree = "<group>"; };
CC98B19F2E17370900A4EDAB /* FileProxyView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileProxyView.swift; sourceTree = "<group>"; };
CC98B1A02E17370900A4EDAB /* ProcessSelectionView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessSelectionView.swift; sourceTree = "<group>"; };
CC98B1A12E17370900A4EDAB /* RealtimeVADView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RealtimeVADView.swift; sourceTree = "<group>"; };
CC98B1A22E17370900A4EDAB /* RecordingIndicator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RecordingIndicator.swift; sourceTree = "<group>"; };
CC98B1A32E17370900A4EDAB /* RecordingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RecordingView.swift; sourceTree = "<group>"; };
CC98B1A42E17370900A4EDAB /* RootView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RootView.swift; sourceTree = "<group>"; };
F431728C2BF68C0C00D918A3 /* AudioRecordingPermission.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioRecordingPermission.swift; sourceTree = "<group>"; };
F431728E2BF691D900D918A3 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist; path = Info.plist; sourceTree = "<group>"; };
F431728F2BF6A92900D918A3 /* AudioProcessController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioProcessController.swift; sourceTree = "<group>"; };
F43172912BF6ABB000D918A3 /* ProcessSelectionView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessSelectionView.swift; sourceTree = "<group>"; };
F43172932BF6B01000D918A3 /* ProcessTap.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProcessTap.swift; sourceTree = "<group>"; };
F43172952BF787C300D918A3 /* CoreAudioUtils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoreAudioUtils.swift; sourceTree = "<group>"; };
F43172992BF7A51A00D918A3 /* RecordingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RecordingView.swift; sourceTree = "<group>"; };
F431729B2BF7B66A00D918A3 /* FileProxyView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileProxyView.swift; sourceTree = "<group>"; };
F431729E2BF7BB0E00D918A3 /* Main.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Main.xcconfig; sourceTree = "<group>"; };
F43172A02BF7BF1300D918A3 /* RecordingIndicator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RecordingIndicator.swift; sourceTree = "<group>"; };
F47AD93E2BF5B61E005B75AC /* AudioCap.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = AudioCap.app; sourceTree = BUILT_PRODUCTS_DIR; };
F47AD9412BF5B61E005B75AC /* AudioCapApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioCapApp.swift; sourceTree = "<group>"; };
F47AD9432BF5B61E005B75AC /* RootView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RootView.swift; sourceTree = "<group>"; };
F47AD9452BF5B61F005B75AC /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = "<group>"; };
F47AD9482BF5B61F005B75AC /* Preview Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = "Preview Assets.xcassets"; sourceTree = "<group>"; };
F47AD94A2BF5B61F005B75AC /* AudioCap.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = AudioCap.entitlements; sourceTree = "<group>"; };
Expand All @@ -51,6 +57,19 @@
/* End PBXFrameworksBuildPhase section */

/* Begin PBXGroup section */
CC98B1A52E17370900A4EDAB /* Views */ = {
isa = PBXGroup;
children = (
CC98B19F2E17370900A4EDAB /* FileProxyView.swift */,
CC98B1A02E17370900A4EDAB /* ProcessSelectionView.swift */,
CC98B1A12E17370900A4EDAB /* RealtimeVADView.swift */,
CC98B1A22E17370900A4EDAB /* RecordingIndicator.swift */,
CC98B1A32E17370900A4EDAB /* RecordingView.swift */,
CC98B1A42E17370900A4EDAB /* RootView.swift */,
);
path = Views;
sourceTree = "<group>";
};
F431729D2BF7BB0600D918A3 /* Config */ = {
isa = PBXGroup;
children = (
Expand All @@ -64,6 +83,8 @@
children = (
F431728C2BF68C0C00D918A3 /* AudioRecordingPermission.swift */,
F431728F2BF6A92900D918A3 /* AudioProcessController.swift */,
CC537FC42E09E7CA00503A96 /* RealtimeAudioMonitor.swift */,
CC537FC62E09E7D500503A96 /* VoiceActivityDetector.swift */,
F43172932BF6B01000D918A3 /* ProcessTap.swift */,
F43172952BF787C300D918A3 /* CoreAudioUtils.swift */,
);
Expand Down Expand Up @@ -92,11 +113,7 @@
F431729D2BF7BB0600D918A3 /* Config */,
F431729F2BF7BD4700D918A3 /* ProcessTap */,
F47AD9412BF5B61E005B75AC /* AudioCapApp.swift */,
F47AD9432BF5B61E005B75AC /* RootView.swift */,
F43172912BF6ABB000D918A3 /* ProcessSelectionView.swift */,
F43172992BF7A51A00D918A3 /* RecordingView.swift */,
F431729B2BF7B66A00D918A3 /* FileProxyView.swift */,
F43172A02BF7BF1300D918A3 /* RecordingIndicator.swift */,
CC98B1A52E17370900A4EDAB /* Views */,
F431728E2BF691D900D918A3 /* Info.plist */,
F47AD9452BF5B61F005B75AC /* Assets.xcassets */,
F47AD94A2BF5B61F005B75AC /* AudioCap.entitlements */,
Expand Down Expand Up @@ -141,7 +158,7 @@
attributes = {
BuildIndependentTargetsInParallel = 1;
LastSwiftUpdateCheck = 1530;
LastUpgradeCheck = 1530;
LastUpgradeCheck = 1640;
TargetAttributes = {
F47AD93D2BF5B61E005B75AC = {
CreatedOnToolsVersion = 15.3;
Expand Down Expand Up @@ -183,16 +200,19 @@
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
CC98B1A62E17370900A4EDAB /* ProcessSelectionView.swift in Sources */,
CC98B1A72E17370900A4EDAB /* RootView.swift in Sources */,
CC98B1A82E17370900A4EDAB /* RecordingIndicator.swift in Sources */,
CC98B1A92E17370900A4EDAB /* FileProxyView.swift in Sources */,
CC98B1AA2E17370900A4EDAB /* RecordingView.swift in Sources */,
CC98B1AB2E17370900A4EDAB /* RealtimeVADView.swift in Sources */,
CC537FC52E09E7CA00503A96 /* RealtimeAudioMonitor.swift in Sources */,
F43172902BF6A92900D918A3 /* AudioProcessController.swift in Sources */,
F43172942BF6B01000D918A3 /* ProcessTap.swift in Sources */,
F43172922BF6ABB000D918A3 /* ProcessSelectionView.swift in Sources */,
F47AD9442BF5B61E005B75AC /* RootView.swift in Sources */,
CC537FC72E09E7D500503A96 /* VoiceActivityDetector.swift in Sources */,
F47AD9422BF5B61E005B75AC /* AudioCapApp.swift in Sources */,
F431728D2BF68C0C00D918A3 /* AudioRecordingPermission.swift in Sources */,
F431729A2BF7A51A00D918A3 /* RecordingView.swift in Sources */,
F43172A12BF7BF1300D918A3 /* RecordingIndicator.swift in Sources */,
F43172962BF787C300D918A3 /* CoreAudioUtils.swift in Sources */,
F431729C2BF7B66A00D918A3 /* FileProxyView.swift in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -234,7 +254,9 @@
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
DEAD_CODE_STRIPPING = YES;
DEBUG_INFORMATION_FORMAT = dwarf;
DEVELOPMENT_TEAM = BD72FKWLAY;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_TESTABILITY = YES;
ENABLE_USER_SCRIPT_SANDBOXING = YES;
Expand Down Expand Up @@ -299,7 +321,9 @@
CLANG_WARN_UNREACHABLE_CODE = YES;
CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
COPY_PHASE_STRIP = NO;
DEAD_CODE_STRIPPING = YES;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
DEVELOPMENT_TEAM = BD72FKWLAY;
ENABLE_NS_ASSERTIONS = NO;
ENABLE_STRICT_OBJC_MSGSEND = YES;
ENABLE_USER_SCRIPT_SANDBOXING = YES;
Expand Down Expand Up @@ -328,8 +352,8 @@
CODE_SIGN_ENTITLEMENTS = AudioCap/AudioCap.entitlements;
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
DEAD_CODE_STRIPPING = YES;
DEVELOPMENT_ASSET_PATHS = "\"AudioCap/Preview Content\"";
DEVELOPMENT_TEAM = 8C7439RJLG;
ENABLE_HARDENED_RUNTIME = YES;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
Expand All @@ -354,8 +378,8 @@
CODE_SIGN_ENTITLEMENTS = AudioCap/AudioCap.entitlements;
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
DEAD_CODE_STRIPPING = YES;
DEVELOPMENT_ASSET_PATHS = "\"AudioCap/Preview Content\"";
DEVELOPMENT_TEAM = 8C7439RJLG;
ENABLE_HARDENED_RUNTIME = YES;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
Expand Down
2 changes: 1 addition & 1 deletion AudioCap.xcodeproj/xcshareddata/xcschemes/AudioCap.xcscheme
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<Scheme
LastUpgradeVersion = "1530"
LastUpgradeVersion = "1640"
version = "1.7">
<BuildAction
parallelizeBuildables = "YES"
Expand Down
117 changes: 117 additions & 0 deletions AudioCap/ProcessTap/RealtimeAudioMonitor.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
import SwiftUI
import AudioToolbox
import AVFoundation
import OSLog

/// Real-time audio monitor for Voice Activity Detection
@Observable
final class RealtimeAudioMonitor {

let process: AudioProcess
private let logger: Logger
private let vad = VoiceActivityDetector()

@ObservationIgnored
private var tap: ProcessTap?

private(set) var isMonitoring = false
private(set) var errorMessage: String?

init(process: AudioProcess) {
self.process = process
self.logger = Logger(subsystem: kAppSubsystem, category: "\(String(describing: RealtimeAudioMonitor.self))(\(process.name))")
}

/// Get the VAD instance for UI binding
var voiceActivityDetector: VoiceActivityDetector {
return vad
}

/// Start real-time monitoring
@MainActor
func startMonitoring() throws {
guard !isMonitoring else {
logger.warning("Already monitoring")
return
}

logger.debug("Starting real-time monitoring for \(self.process.name)")

errorMessage = nil

// Create a new tap for monitoring (separate from recording)
let monitorTap = ProcessTap(process: process, muteWhenRunning: false)
self.tap = monitorTap

// Activate the tap
monitorTap.activate()

if let error = monitorTap.errorMessage {
throw error
}

// Start the monitoring process
let queue = DispatchQueue(label: "RealtimeAudioMonitor", qos: .userInitiated)

guard let streamDescription = monitorTap.tapStreamDescription else {
throw "Tap stream description not available"
}

var mutableStreamDescription = streamDescription
guard let format = AVAudioFormat(streamDescription: &mutableStreamDescription) else {
throw "Failed to create AVAudioFormat"
}

logger.info("Monitoring with audio format: \(format)")

try monitorTap.run(on: queue) { [weak self] inNow, inInputData, inInputTime, outOutputData, inOutputTime in
guard let self = self else { return }

do {
// Create PCM buffer from input data
guard let buffer = AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: inInputData, deallocator: nil) else {
throw "Failed to create PCM buffer"
}

// Process buffer through VAD (ensure this runs on main thread for @Observable updates)
DispatchQueue.main.async {
self.vad.processAudioBuffer(buffer)
}

} catch {
self.logger.error("Error processing audio buffer: \(error)")
}
} invalidationHandler: { [weak self] tap in
guard let self = self else { return }
DispatchQueue.main.async {
self.handleMonitoringStop()
}
}

isMonitoring = true
logger.debug("Real-time monitoring started successfully")
}

/// Stop real-time monitoring
@MainActor
func stopMonitoring() {
guard isMonitoring else { return }

logger.debug("Stopping real-time monitoring")

tap?.invalidate()
tap = nil

handleMonitoringStop()
}

private func handleMonitoringStop() {
isMonitoring = false
vad.reset()
logger.debug("Real-time monitoring stopped")
}

deinit {
tap?.invalidate()
}
}
Loading