Добавление упреждающих сообщений в агент голосовой связи в реальном времени (предварительная версия)

async for event in conn:
    await self._handle_event(event)

await foreach (SessionUpdate serverEvent in session.GetUpdatesAsync(cancellationToken))
{
    await HandleSessionUpdateAsync(serverEvent, cancellationToken);
}

session.receiveEvents()
    .subscribe(
        event -> handleServerEvent(event, audioProcessor),
        error -> System.err.println("Error: " + error.getMessage())
    );

const subscription = session.subscribe({
    onSessionUpdated: async (event, context) => {
        await handleEvent(event);
    }
});

self.greeting_sent = False

private bool _greetingSent = false;

private AtomicBoolean greetingSent = new AtomicBoolean(false);

let greetingSent = false;

if event.type == ServerEventType.SESSION_UPDATED:
    if not self.greeting_sent:
        # ... send proactive message ...
        self.greeting_sent = True

if (serverEvent is SessionUpdateSessionUpdated)
{
    if (!_greetingSent)
    {
        // ... send proactive message ...
        _greetingSent = true;
    }
}

if (event.getType() == ServerEventType.SESSION_UPDATED) {
    if (!greetingSent.getAndSet(true)) {
        // ... send proactive message ...
    }
}

if (event.type === "session.updated") {
    if (!greetingSent) {
        // ... send proactive message ...
        greetingSent = true;
    }
}

from azure.ai.voicelive.models import (
    ResponseCreateParams,
    AssistantMessageItem,
    OutputTextContentPart,
)

# Send a pre-generated assistant greeting
await conn.response.create(
    response=ResponseCreateParams(
        pre_generated_assistant_message=AssistantMessageItem(
            content=[
                OutputTextContentPart(
                    text="Hi Lisa, welcome back! How can I assist you today?"
                )
            ]
        )
    )
)

using System.Text.Json;

// Send a pre-generated assistant greeting
var greeting = "Hi Lisa, welcome back! How can I assist you today?";
var responseCreatePayload = new
{
    type = "response.create",
    response = new
    {
        pre_generated_assistant_message = new
        {
            type = "message",
            role = "assistant",
            content = new[]
            {
                new { type = "text", text = greeting }
            }
        }
    }
};
BinaryData eventData = BinaryData.FromObjectAsJson(responseCreatePayload);
await session.SendCommandAsync(eventData, cancellationToken).ConfigureAwait(false);

import com.azure.ai.voicelive.models.*;
import java.util.Arrays;

// Send a pre-generated assistant greeting
OutputTextContentPart textContent = new OutputTextContentPart(
    "Hi Lisa, welcome back! How can I assist you today?");
AssistantMessageItem assistantMessage = new AssistantMessageItem(Arrays.asList(textContent));

ResponseCreateParams responseParams = new ResponseCreateParams()
    .setPreGeneratedAssistantMessage(assistantMessage);

session.sendEvent(new ClientEventResponseCreate().setResponse(responseParams)).subscribe();

// Send a pre-generated assistant greeting
await session.sendEvent({
    type: "response.create",
    response: {
        preGeneratedAssistantMessage: {
            content: [
                {
                    type: "text",
                    text: "Hi Lisa, welcome back! How can I assist you today?"
                }
            ]
        }
    }
});

from azure.ai.voicelive.models import (
    MessageItem,
    InputTextContentPart,
)

# Add an instruction telling the LLM to greet the user
await conn.conversation.item.create(
    item=MessageItem(
        role="system",
        content=[
            InputTextContentPart(
                text="Say something to welcome the user."
            )
        ]
    )
)

# Trigger LLM-generated greeting
await conn.response.create()

using Azure.AI.VoiceLive;

// Add an instruction telling the LLM to greet the user
var systemMessage = new SystemMessageItem(
    new InputTextContentPart("Say something to welcome the user."));
await session.AddItemAsync(systemMessage, cancellationToken).ConfigureAwait(false);

// Trigger LLM-generated greeting
await session.StartResponseAsync().ConfigureAwait(false);

import com.azure.ai.voicelive.models.*;
import java.util.Arrays;

// Add an instruction telling the LLM to greet the user
InputTextContentPart textContent = new InputTextContentPart(
    "Say something to welcome the user.");
SystemMessageItem systemMessage = new SystemMessageItem(Arrays.asList(textContent));

session.sendEvent(new ClientEventConversationItemCreate().setItem(systemMessage)).subscribe();

// Trigger LLM-generated greeting
session.sendEvent(new ClientEventResponseCreate()).subscribe();

// Add an instruction telling the LLM to greet the user
await session.addConversationItem({
    type: "message",
    role: "system",
    content: [
        {
            type: "input_text",
            text: "Say something to welcome the user."
        }
    ]
});

// Trigger LLM-generated greeting
await session.sendEvent({
    type: "response.create"
});

async def _handle_event(self, event):
    conn = self.connection
    ap = self.audio_processor

    if event.type == ServerEventType.SESSION_UPDATED:
        print("Session is ready.")
        self.session_ready = True

        # Start microphone capture
        ap.start_capture()

        # Trigger greeting once
        if not self.greeting_sent:
            self.greeting_sent = True

            # ----- Option 1: Pre-generated greeting -----
            # await send_pre_generated_greeting(conn)

            # ----- Option 2: LLM-generated greeting -----
            # await send_llm_generated_greeting(conn)

    elif event.type == ServerEventType.INPUT_AUDIO_BUFFER_SPEECH_STARTED:
        print("🎤 Listening...")
        ap.skip_pending_audio()

    elif event.type == ServerEventType.RESPONSE_AUDIO_DELTA:
        ap.queue_audio(event.delta)

    elif event.type == ServerEventType.RESPONSE_AUDIO_DONE:
        print("🎤 Ready for next input...")

private async Task HandleSessionUpdateAsync(SessionUpdate serverEvent, CancellationToken cancellationToken)
{
    if (serverEvent is SessionUpdateSessionUpdated sessionUpdated)
    {
        Console.WriteLine("Session is ready.");
        _sessionReady = true;

        // Start microphone capture
        await _audioProcessor.StartCaptureAsync();

        // Trigger greeting once
        if (!_greetingSent)
        {
            _greetingSent = true;

            // ----- Option 1: Pre-generated greeting -----
            // await SendPreGeneratedGreetingAsync(_session, cancellationToken);

            // ----- Option 2: LLM-generated greeting -----
            // await SendLlmGeneratedGreetingAsync(_session, cancellationToken);
        }
    }
    else if (serverEvent is SessionUpdateInputAudioBufferSpeechStarted)
    {
        Console.WriteLine("🎤 Listening...");
        await _audioProcessor.StopPlaybackAsync();
    }
    else if (serverEvent is SessionUpdateResponseAudioDelta audioDelta)
    {
        await _audioProcessor.QueueAudioAsync(audioDelta.Delta.ToArray());
    }
    else if (serverEvent is SessionUpdateResponseAudioDone)
    {
        Console.WriteLine("🎤 Ready for next input...");
    }
}

private void handleServerEvent(SessionUpdate event, AudioProcessor audioProcessor) {
    ServerEventType eventType = event.getType();

    if (eventType == ServerEventType.SESSION_UPDATED) {
        System.out.println("Session is ready.");
        sessionReady = true;

        // Start microphone capture
        audioProcessor.startCapture();

        // Trigger greeting once
        if (!greetingSent.getAndSet(true)) {

            // ----- Option 1: Pre-generated greeting -----
            // sendPreGeneratedGreeting(session);

            // ----- Option 2: LLM-generated greeting -----
            // sendLlmGeneratedGreeting(session);
        }
    } else if (eventType == ServerEventType.INPUT_AUDIO_BUFFER_SPEECH_STARTED) {
        System.out.println("🎤 Listening...");
        audioProcessor.skipPendingAudio();
    } else if (eventType == ServerEventType.RESPONSE_AUDIO_DELTA) {
        SessionUpdateResponseAudioDelta audioEvent = (SessionUpdateResponseAudioDelta) event;
        audioProcessor.queueAudio(audioEvent.getDelta());
    } else if (eventType == ServerEventType.RESPONSE_AUDIO_DONE) {
        System.out.println("🎤 Ready for next input...");
    }
}

const subscription = session.subscribe({
    onSessionUpdated: async (event, context) => {
        console.log("Session is ready.");
        sessionReady = true;

        // Start microphone capture
        startMicrophoneCapture();

        // Trigger greeting once
        if (!greetingSent) {
            greetingSent = true;

            // ----- Option 1: Pre-generated greeting -----
            // await sendPreGeneratedGreeting(session);

            // ----- Option 2: LLM-generated greeting -----
            // await sendLlmGeneratedGreeting(session);
        }
    },

    onInputAudioBufferSpeechStarted: async (event, context) => {
        console.log("🎤 Listening...");
        skipPendingAudio();
    },

    onResponseAudioDelta: async (event, context) => {
        queueAudio(event.delta);
    },

    onResponseAudioDone: async (event, context) => {
        console.log("🎤 Ready for next input...");
    }
});

async def send_pre_generated_greeting(conn):
    try:
        await conn.response.create(
            response=ResponseCreateParams(
                pre_generated_assistant_message=AssistantMessageItem(
                    content=[
                        OutputTextContentPart(
                            text="Welcome! I'm here to help you get started."
                        )
                    ]
                )
            )
        )
    except Exception as e:
        print(f"Failed to send pre-generated greeting: {e}")

async def send_llm_generated_greeting(conn):
    try:
        await conn.conversation.item.create(
            item=MessageItem(
                role="system",
                content=[
                    InputTextContentPart(
                        text="Greet the user warmly and briefly explain how you can help."
                    )
                ]
            )
        )
        await conn.response.create()
    except Exception as e:
        print(f"Failed to send LLM-generated greeting: {e}")

private async Task SendPreGeneratedGreetingAsync(
    VoiceLiveSession session, 
    CancellationToken cancellationToken)
{
    try
    {
        var greeting = "Welcome! I'm here to help you get started.";
        var responseCreatePayload = new
        {
            type = "response.create",
            response = new
            {
                pre_generated_assistant_message = new
                {
                    type = "message",
                    role = "assistant",
                    content = new[]
                    {
                        new { type = "text", text = greeting }
                    }
                }
            }
        };
        BinaryData eventData = BinaryData.FromObjectAsJson(responseCreatePayload);
        await session.SendCommandAsync(eventData, cancellationToken).ConfigureAwait(false);
    }
    catch (Exception ex)
    {
        Console.WriteLine($"Failed to send pre-generated greeting: {ex.Message}");
    }
}

private async Task SendLlmGeneratedGreetingAsync(
    VoiceLiveSession session, 
    CancellationToken cancellationToken)
{
    try
    {
        var systemMessage = new SystemMessageItem(
            new InputTextContentPart("Greet the user warmly and briefly explain how you can help."));
        await session.AddItemAsync(systemMessage, cancellationToken).ConfigureAwait(false);
        await session.StartResponseAsync().ConfigureAwait(false);
    }
    catch (Exception ex)
    {
        Console.WriteLine($"Failed to send LLM-generated greeting: {ex.Message}");
    }
}

private void sendPreGeneratedGreeting(VoiceLiveSessionAsyncClient session) {
    try {
        OutputTextContentPart textContent = new OutputTextContentPart(
            "Welcome! I'm here to help you get started.");
        AssistantMessageItem assistantMessage = new AssistantMessageItem(Arrays.asList(textContent));

        ResponseCreateParams responseParams = new ResponseCreateParams()
            .setPreGeneratedAssistantMessage(assistantMessage);

        session.sendEvent(new ClientEventResponseCreate().setResponse(responseParams))
            .doOnError(error -> System.err.println("Failed to send pre-generated greeting: " + error.getMessage()))
            .subscribe();
    } catch (Exception e) {
        System.err.println("Failed to send pre-generated greeting: " + e.getMessage());
    }
}

private void sendLlmGeneratedGreeting(VoiceLiveSessionAsyncClient session) {
    try {
        InputTextContentPart textContent = new InputTextContentPart(
            "Greet the user warmly and briefly explain how you can help.");
        SystemMessageItem systemMessage = new SystemMessageItem(Arrays.asList(textContent));

        session.sendEvent(new ClientEventConversationItemCreate().setItem(systemMessage))
            .doOnError(error -> System.err.println("Failed to add conversation item: " + error.getMessage()))
            .subscribe();
        session.sendEvent(new ClientEventResponseCreate())
            .doOnError(error -> System.err.println("Failed to create response: " + error.getMessage()))
            .subscribe();
    } catch (Exception e) {
        System.err.println("Failed to send LLM-generated greeting: " + e.getMessage());
    }
}

async function sendPreGeneratedGreeting(session) {
    try {
        await session.sendEvent({
            type: "response.create",
            response: {
                preGeneratedAssistantMessage: {
                    content: [
                        {
                            type: "text",
                            text: "Welcome! I'm here to help you get started."
                        }
                    ]
                }
            }
        });
    } catch (error) {
        console.error("Failed to send pre-generated greeting:", error.message);
    }
}

async function sendLlmGeneratedGreeting(session) {
    try {
        await session.addConversationItem({
            type: "message",
            role: "system",
            content: [
                {
                    type: "input_text",
                    text: "Greet the user warmly and briefly explain how you can help."
                }
            ]
        });
        await session.sendEvent({
            type: "response.create"
        });
    } catch (error) {
        console.error("Failed to send LLM-generated greeting:", error.message);
    }
}