Bases: RealtimeClientBase
(Experimental) Client for OpenAI Realtime API.
(Experimental) Client for OpenAI Realtime API.
PARAMETER | DESCRIPTION |
llm_config | The config for the client. TYPE: dict[str, Any] |
logger | the logger to use for logging events TYPE: Optional[Logger] DEFAULT: None |
Source code in autogen/agentchat/realtime/experimental/clients/oai/base_client.py
| def __init__(
self,
*,
llm_config: dict[str, Any],
logger: Optional[Logger] = None,
) -> None:
"""(Experimental) Client for OpenAI Realtime API.
Args:
llm_config: The config for the client.
logger: the logger to use for logging events
"""
super().__init__()
self._llm_config = llm_config
self._logger = logger
self._connection: Optional["AsyncRealtimeConnection"] = None
self.config = llm_config["config_list"][0]
# model is passed to self._client.beta.realtime.connect function later
self._model: str = self.config["model"]
self._voice: str = self.config.get("voice", "alloy")
self._temperature: float = llm_config.get("temperature", 0.8) # type: ignore[union-attr]
self._client: Optional["AsyncOpenAI"] = None
|
config instance-attribute
config = llm_config['config_list'][0]
logger property
Get the logger for the OpenAI Realtime API.
connection property
Get the OpenAI WebSocket connection.
add_event async
Source code in autogen/agentchat/realtime/experimental/clients/realtime_client.py
| async def add_event(self, event: Optional[RealtimeEvent]):
await self._eventQueue.put(event)
|
get_event async
Source code in autogen/agentchat/realtime/experimental/clients/realtime_client.py
| async def get_event(self) -> Optional[RealtimeEvent]:
return await self._eventQueue.get()
|
queue_input_audio_buffer_delta(audio)
queue InputAudioBufferDelta.
PARAMETER | DESCRIPTION |
audio | TYPE: str |
Source code in autogen/agentchat/realtime/experimental/clients/realtime_client.py
| async def queue_input_audio_buffer_delta(self, audio: str) -> None:
"""queue InputAudioBufferDelta.
Args:
audio (str): The audio.
"""
await self.add_event(InputAudioBufferDelta(delta=audio, item_id=None, raw_message=dict()))
|
send_function_result async
send_function_result(call_id, result)
Send the result of a function call to the OpenAI Realtime API.
PARAMETER | DESCRIPTION |
call_id | The ID of the function call. TYPE: str |
result | The result of the function call. TYPE: str |
Source code in autogen/agentchat/realtime/experimental/clients/oai/base_client.py
| async def send_function_result(self, call_id: str, result: str) -> None:
"""Send the result of a function call to the OpenAI Realtime API.
Args:
call_id (str): The ID of the function call.
result (str): The result of the function call.
"""
await self.connection.conversation.item.create(
item={
"type": "function_call_output",
"call_id": call_id,
"output": result,
},
)
await self.connection.response.create()
|
send_text async
Send a text message to the OpenAI Realtime API.
PARAMETER | DESCRIPTION |
role | TYPE: str |
text | TYPE: str |
Source code in autogen/agentchat/realtime/experimental/clients/oai/base_client.py
| async def send_text(self, *, role: Role, text: str) -> None:
"""Send a text message to the OpenAI Realtime API.
Args:
role (str): The role of the message.
text (str): The text of the message.
"""
await self.connection.response.cancel()
await self.connection.conversation.item.create(
item={"type": "message", "role": role, "content": [{"type": "input_text", "text": text}]}
)
await self.connection.response.create()
|
send_audio async
Send audio to the OpenAI Realtime API.
PARAMETER | DESCRIPTION |
audio | TYPE: str |
Source code in autogen/agentchat/realtime/experimental/clients/oai/base_client.py
| async def send_audio(self, audio: str) -> None:
"""Send audio to the OpenAI Realtime API.
Args:
audio (str): The audio to send.
"""
await self.queue_input_audio_buffer_delta(audio)
await self.connection.input_audio_buffer.append(audio=audio)
|
truncate_audio async
truncate_audio(audio_end_ms, content_index, item_id)
Truncate audio in the OpenAI Realtime API.
PARAMETER | DESCRIPTION |
audio_end_ms | The end of the audio to truncate. TYPE: int |
content_index | The index of the content to truncate. TYPE: int |
item_id | The ID of the item to truncate. TYPE: str |
Source code in autogen/agentchat/realtime/experimental/clients/oai/base_client.py
| async def truncate_audio(self, audio_end_ms: int, content_index: int, item_id: str) -> None:
"""Truncate audio in the OpenAI Realtime API.
Args:
audio_end_ms (int): The end of the audio to truncate.
content_index (int): The index of the content to truncate.
item_id (str): The ID of the item to truncate.
"""
await self.connection.conversation.item.truncate(
audio_end_ms=audio_end_ms, content_index=content_index, item_id=item_id
)
|
session_update async
session_update(session_options)
Send a session update to the OpenAI Realtime API.
PARAMETER | DESCRIPTION |
session_options | The session options to update. TYPE: dict[str, Any] |
Source code in autogen/agentchat/realtime/experimental/clients/oai/base_client.py
| async def session_update(self, session_options: dict[str, Any]) -> None:
"""Send a session update to the OpenAI Realtime API.
Args:
session_options (dict[str, Any]): The session options to update.
"""
logger = self.logger
logger.info(f"Sending session update: {session_options}")
await self.connection.session.update(session=session_options) # type: ignore[arg-type]
logger.info("Sending session update finished")
|
connect async
Connect to the OpenAI Realtime API.
Source code in autogen/agentchat/realtime/experimental/clients/oai/base_client.py
| @asynccontextmanager
async def connect(self) -> AsyncGenerator[None, None]:
"""Connect to the OpenAI Realtime API."""
try:
if not self._client:
self._client = AsyncOpenAI(
api_key=self.config.get("api_key", None),
organization=self.config.get("organization", None),
project=self.config.get("project", None),
base_url=self.config.get("base_url", None),
websocket_base_url=self.config.get("websocket_base_url", None),
timeout=self.config.get("timeout", NOT_GIVEN),
max_retries=self.config.get("max_retries", DEFAULT_MAX_RETRIES),
default_headers=self.config.get("default_headers", None),
default_query=self.config.get("default_query", None),
)
async with self._client.beta.realtime.connect(
model=self._model,
) as self._connection:
await self._initialize_session()
yield
finally:
self._connection = None
|
read_events async
Read messages from the OpenAI Realtime API.
Source code in autogen/agentchat/realtime/experimental/clients/oai/base_client.py
| async def read_events(self) -> AsyncGenerator[RealtimeEvent, None]:
"""Read messages from the OpenAI Realtime API."""
if self._connection is None:
raise RuntimeError("Client is not connected, call connect() first.")
try:
async for event in self._read_events():
yield event
finally:
self._connection = None
|
get_factory classmethod
get_factory(llm_config, logger, **kwargs)
Create a Realtime API client.
PARAMETER | DESCRIPTION |
llm_config | The config for the client. TYPE: dict[str, Any] |
logger | The logger to use for logging events. TYPE: Logger |
kwargs | TYPE: Any DEFAULT: {} |
Source code in autogen/agentchat/realtime/experimental/clients/oai/base_client.py
| @classmethod
def get_factory(
cls, llm_config: dict[str, Any], logger: Logger, **kwargs: Any
) -> Optional[Callable[[], "RealtimeClientProtocol"]]:
"""Create a Realtime API client.
Args:
llm_config: The config for the client.
logger: The logger to use for logging events.
kwargs: Additional arguments.
Returns:
RealtimeClientProtocol: The Realtime API client is returned if the model matches the pattern
"""
if llm_config["config_list"][0].get("api_type", "openai") == "openai" and list(kwargs.keys()) == []:
return lambda: OpenAIRealtimeClient(llm_config=llm_config, logger=logger, **kwargs)
return None
|