Spaces:
Running
Running
Merge deployment-updates into main, taking all changes from deployment-updates
Browse files
README.md
CHANGED
|
@@ -5,7 +5,8 @@ colorFrom: blue
|
|
| 5 |
colorTo: purple
|
| 6 |
sdk: docker
|
| 7 |
pinned: false
|
| 8 |
-
|
|
|
|
| 9 |
---
|
| 10 |
|
| 11 |
# Multimodal Live API - Web console
|
|
|
|
| 5 |
colorTo: purple
|
| 6 |
sdk: docker
|
| 7 |
pinned: false
|
| 8 |
+
license: apache-2.0
|
| 9 |
+
short_description: Gemini Multimodal Live API Console
|
| 10 |
---
|
| 11 |
|
| 12 |
# Multimodal Live API - Web console
|
src/components/control-tray/ControlTray.tsx
CHANGED
|
@@ -244,24 +244,22 @@ function ControlTray({
|
|
| 244 |
ref={connectButtonRef}
|
| 245 |
className={cn("action-button connect-toggle", { connected })}
|
| 246 |
onClick={async () => {
|
| 247 |
-
console.log('
|
| 248 |
try {
|
| 249 |
if (connected) {
|
| 250 |
-
console.log('๐ด
|
| 251 |
await disconnect();
|
| 252 |
-
console.log('โ
Disconnected successfully');
|
| 253 |
} else {
|
| 254 |
-
console.log('๐ Starting connection...');
|
| 255 |
-
console.log('๐ฑ Device info:', { isIOSDevice, isSafari });
|
| 256 |
|
| 257 |
-
|
| 258 |
-
console.log('๐ Calling connect()...');
|
| 259 |
await connect();
|
| 260 |
-
console.log('โ
Connected successfully');
|
| 261 |
}
|
| 262 |
} catch (err) {
|
| 263 |
-
console.error('โ
|
| 264 |
-
// Here you could add UI feedback about the error
|
| 265 |
}
|
| 266 |
}}
|
| 267 |
>
|
|
|
|
| 244 |
ref={connectButtonRef}
|
| 245 |
className={cn("action-button connect-toggle", { connected })}
|
| 246 |
onClick={async () => {
|
| 247 |
+
console.log('๐ฎ ControlTray: Connection button clicked');
|
| 248 |
try {
|
| 249 |
if (connected) {
|
| 250 |
+
console.log('๐ด ControlTray: Initiating disconnect...');
|
| 251 |
await disconnect();
|
| 252 |
+
console.log('โ
ControlTray: Disconnected successfully');
|
| 253 |
} else {
|
| 254 |
+
console.log('๐ ControlTray: Starting connection process...');
|
| 255 |
+
console.log('๐ฑ ControlTray: Device info:', { isIOSDevice, isSafari });
|
| 256 |
|
| 257 |
+
console.log('๐ ControlTray: Calling LiveAPIContext.connect()...');
|
|
|
|
| 258 |
await connect();
|
| 259 |
+
console.log('โ
ControlTray: Connected successfully via LiveAPIContext');
|
| 260 |
}
|
| 261 |
} catch (err) {
|
| 262 |
+
console.error('โ ControlTray: Connection error:', err);
|
|
|
|
| 263 |
}
|
| 264 |
}}
|
| 265 |
>
|
src/contexts/LiveAPIContext.tsx
CHANGED
|
@@ -28,7 +28,6 @@ export const LiveAPIProvider: FC<LiveAPIProviderProps> = ({
|
|
| 28 |
url = `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`,
|
| 29 |
children,
|
| 30 |
}) => {
|
| 31 |
-
console.log('๐ Initializing LiveAPIProvider with URL:', url);
|
| 32 |
const liveAPI = useLiveAPI({ url });
|
| 33 |
|
| 34 |
return (
|
|
@@ -39,9 +38,9 @@ export const LiveAPIProvider: FC<LiveAPIProviderProps> = ({
|
|
| 39 |
};
|
| 40 |
|
| 41 |
export const useLiveAPIContext = () => {
|
|
|
|
| 42 |
const context = useContext(LiveAPIContext);
|
| 43 |
if (!context) {
|
| 44 |
-
console.error('โ LiveAPIContext used outside of LiveAPIProvider');
|
| 45 |
throw new Error("useLiveAPIContext must be used within a LiveAPIProvider");
|
| 46 |
}
|
| 47 |
console.log('โ
LiveAPIContext successfully retrieved');
|
|
|
|
| 28 |
url = `${window.location.protocol === 'https:' ? 'wss:' : 'ws:'}//${window.location.host}/ws`,
|
| 29 |
children,
|
| 30 |
}) => {
|
|
|
|
| 31 |
const liveAPI = useLiveAPI({ url });
|
| 32 |
|
| 33 |
return (
|
|
|
|
| 38 |
};
|
| 39 |
|
| 40 |
export const useLiveAPIContext = () => {
|
| 41 |
+
console.log('๐ฏ LiveAPIContext: Hook being accessed');
|
| 42 |
const context = useContext(LiveAPIContext);
|
| 43 |
if (!context) {
|
|
|
|
| 44 |
throw new Error("useLiveAPIContext must be used within a LiveAPIProvider");
|
| 45 |
}
|
| 46 |
console.log('โ
LiveAPIContext successfully retrieved');
|
src/lib/multimodal-live-client.ts
CHANGED
|
@@ -89,20 +89,21 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
|
|
| 89 |
connect(config: LiveConfig): Promise<boolean> {
|
| 90 |
console.log('๐ Attempting WebSocket connection to:', this.url);
|
| 91 |
this.config = config;
|
|
|
|
| 92 |
|
| 93 |
const ws = new WebSocket(this.url);
|
| 94 |
|
| 95 |
ws.addEventListener("message", async (evt: MessageEvent) => {
|
| 96 |
console.log('๐จ Received WebSocket message:', evt.data instanceof Blob ? 'Blob data' : evt.data);
|
| 97 |
if (evt.data instanceof Blob) {
|
|
|
|
| 98 |
this.receive(evt.data);
|
| 99 |
} else {
|
| 100 |
-
console.log("
|
| 101 |
}
|
| 102 |
});
|
| 103 |
return new Promise((resolve, reject) => {
|
| 104 |
const onError = (ev: Event) => {
|
| 105 |
-
console.error('โ WebSocket connection error:', ev);
|
| 106 |
this.disconnect(ws);
|
| 107 |
const message = `Could not connect to "${this.url}"`;
|
| 108 |
this.log(`server.${ev.type}`, message);
|
|
@@ -112,10 +113,10 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
|
|
| 112 |
ws.addEventListener("open", (ev: Event) => {
|
| 113 |
console.log('โ
WebSocket connection opened successfully');
|
| 114 |
if (!this.config) {
|
| 115 |
-
console.error('โ Invalid config provided to connect()');
|
| 116 |
reject("Invalid config sent to `connect(config)`");
|
| 117 |
return;
|
| 118 |
}
|
|
|
|
| 119 |
this.log(`client.${ev.type}`, `connected to socket`);
|
| 120 |
this.emit("open");
|
| 121 |
|
|
@@ -124,13 +125,12 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
|
|
| 124 |
const setupMessage: SetupMessage = {
|
| 125 |
setup: this.config,
|
| 126 |
};
|
| 127 |
-
console.log('๐ค Sending setup message:', setupMessage);
|
| 128 |
this._sendDirect(setupMessage);
|
| 129 |
this.log("client.send", "setup");
|
| 130 |
|
| 131 |
ws.removeEventListener("error", onError);
|
| 132 |
ws.addEventListener("close", (ev: CloseEvent) => {
|
| 133 |
-
console.log(
|
| 134 |
this.disconnect(ws);
|
| 135 |
let reason = ev.reason || "";
|
| 136 |
if (reason.toLowerCase().includes("error")) {
|
|
@@ -176,17 +176,20 @@ export class MultimodalLiveClient extends EventEmitter<MultimodalLiveClientEvent
|
|
| 176 |
)) as LiveIncomingMessage;
|
| 177 |
console.log('๐ฅ Received message:', response);
|
| 178 |
if (isToolCallMessage(response)) {
|
|
|
|
| 179 |
this.log("server.toolCall", response);
|
| 180 |
this.emit("toolcall", response.toolCall);
|
| 181 |
return;
|
| 182 |
}
|
| 183 |
if (isToolCallCancellationMessage(response)) {
|
|
|
|
| 184 |
this.log("receive.toolCallCancellation", response);
|
| 185 |
this.emit("toolcallcancellation", response.toolCallCancellation);
|
| 186 |
return;
|
| 187 |
}
|
| 188 |
|
| 189 |
if (isSetupCompleteMessage(response)) {
|
|
|
|
| 190 |
this.log("server.send", "setupComplete");
|
| 191 |
this.emit("setupcomplete");
|
| 192 |
return;
|
|
|
|
| 89 |
connect(config: LiveConfig): Promise<boolean> {
|
| 90 |
console.log('๐ Attempting WebSocket connection to:', this.url);
|
| 91 |
this.config = config;
|
| 92 |
+
console.log('๐ MultimodalLiveClient: Starting WebSocket connection to:', this.url);
|
| 93 |
|
| 94 |
const ws = new WebSocket(this.url);
|
| 95 |
|
| 96 |
ws.addEventListener("message", async (evt: MessageEvent) => {
|
| 97 |
console.log('๐จ Received WebSocket message:', evt.data instanceof Blob ? 'Blob data' : evt.data);
|
| 98 |
if (evt.data instanceof Blob) {
|
| 99 |
+
console.log('๐ฉ MultimodalLiveClient: Received blob message');
|
| 100 |
this.receive(evt.data);
|
| 101 |
} else {
|
| 102 |
+
console.log("non blob message", evt);
|
| 103 |
}
|
| 104 |
});
|
| 105 |
return new Promise((resolve, reject) => {
|
| 106 |
const onError = (ev: Event) => {
|
|
|
|
| 107 |
this.disconnect(ws);
|
| 108 |
const message = `Could not connect to "${this.url}"`;
|
| 109 |
this.log(`server.${ev.type}`, message);
|
|
|
|
| 113 |
ws.addEventListener("open", (ev: Event) => {
|
| 114 |
console.log('โ
WebSocket connection opened successfully');
|
| 115 |
if (!this.config) {
|
|
|
|
| 116 |
reject("Invalid config sent to `connect(config)`");
|
| 117 |
return;
|
| 118 |
}
|
| 119 |
+
console.log('โจ MultimodalLiveClient: WebSocket connection established');
|
| 120 |
this.log(`client.${ev.type}`, `connected to socket`);
|
| 121 |
this.emit("open");
|
| 122 |
|
|
|
|
| 125 |
const setupMessage: SetupMessage = {
|
| 126 |
setup: this.config,
|
| 127 |
};
|
|
|
|
| 128 |
this._sendDirect(setupMessage);
|
| 129 |
this.log("client.send", "setup");
|
| 130 |
|
| 131 |
ws.removeEventListener("error", onError);
|
| 132 |
ws.addEventListener("close", (ev: CloseEvent) => {
|
| 133 |
+
console.log(ev);
|
| 134 |
this.disconnect(ws);
|
| 135 |
let reason = ev.reason || "";
|
| 136 |
if (reason.toLowerCase().includes("error")) {
|
|
|
|
| 176 |
)) as LiveIncomingMessage;
|
| 177 |
console.log('๐ฅ Received message:', response);
|
| 178 |
if (isToolCallMessage(response)) {
|
| 179 |
+
console.log('๐ ๏ธ MultimodalLiveClient: Received tool call');
|
| 180 |
this.log("server.toolCall", response);
|
| 181 |
this.emit("toolcall", response.toolCall);
|
| 182 |
return;
|
| 183 |
}
|
| 184 |
if (isToolCallCancellationMessage(response)) {
|
| 185 |
+
console.log('๐ซ MultimodalLiveClient: Received tool call cancellation');
|
| 186 |
this.log("receive.toolCallCancellation", response);
|
| 187 |
this.emit("toolcallcancellation", response.toolCallCancellation);
|
| 188 |
return;
|
| 189 |
}
|
| 190 |
|
| 191 |
if (isSetupCompleteMessage(response)) {
|
| 192 |
+
console.log('๐ MultimodalLiveClient: Setup complete received');
|
| 193 |
this.log("server.send", "setupComplete");
|
| 194 |
this.emit("setupcomplete");
|
| 195 |
return;
|