Быстрый старт. Воспроизведение аудио на стороне сервера

Начало работы с использованием аудиопотоков с помощью API потоковой передачи звука Службы коммуникации Azure. В этом кратком руководстве предполагается, что вы уже знакомы с API автоматизации вызовов для создания решения автоматической маршрутизации вызовов.

Функции, описанные в этом кратком руководстве, в настоящее время находятся в общедоступной предварительной версии.

MediaStreamingOptions mediaStreamingOptions = new MediaStreamingOptions(MediaStreamingAudioChannel.Unmixed);
 mediaStreamingOptions.TransportUri = new Uri(websocketUri);
 mediaStreamingOptions.EnableBidirectional = true;
 mediaStreamingOptions.AudioFormat = AudioFormat.Pcm24KMono;
 mediaStreamingOptions.EnableDtmfTones = true;
  
var options = new AnswerCallOptions(incomingCallContext, callbackUri) {
  MediaStreamingOptions = mediaStreamingOptions,
};

AnswerCallResult answerCallResult = await client.AnswerCallAsync(options);

MediaStreamingOptions mediaStreamingOptions = new MediaStreamingOptions(MediaStreamingAudioChannel.Unmixed);
 mediaStreamingOptions.TransportUri = new Uri(websocketUri);
 mediaStreamingOptions.EnableBidirectional = true;
 mediaStreamingOptions.AudioFormat = AudioFormat.Pcm24KMono;
 mediaStreamingOptions.EnableDtmfTones = true;

var options = new AnswerCallOptions(incomingCallContext, callbackUri) {
  MediaStreamingOptions = mediaStreamingOptions,
};

AnswerCallResult answerCallResult = await client.AnswerCallAsync(options);

Start media streaming via API call
StartMediaStreamingOptions options = new StartMediaStreamingOptions() {
  OperationContext = "startMediaStreamingContext",
  OperationCallbackUri = eventCallbackUri
};

await callMedia.StartMediaStreamingAsync(options);

StopMediaStreamingOptions options = new StopMediaStreamingOptions() {
  OperationContext = "stopMediaStreamingContext",
  OperationCallbackUri = eventCallbackUri
};

await callMedia.StopMediaStreamingAsync(options);

private async Task StartReceivingFromAcsMediaWebSocket(Websocket websocket) {

  while (webSocket.State == WebSocketState.Open || webSocket.State == WebSocketState.Closed) {
    byte[] receiveBuffer = new byte[2048];
    WebSocketReceiveResult receiveResult = await webSocket.ReceiveAsync(
      new ArraySegment < byte > (receiveBuffer));

    if (receiveResult.MessageType != WebSocketMessageType.Close) {
      string data = Encoding.UTF8.GetString(receiveBuffer).TrimEnd('\0');
      var input = StreamingData.Parse(data);
      if (input is AudioData audioData) {
        // Add your code here to process the received audio chunk
      }
    }
  }
}

{
  "kind": "AudioMetadata",
  "audioMetadata": {
    "subscriptionId": "89e8cb59-b991-48b0-b154-1db84f16a077",
    "encoding": "PCM",
    "sampleRate": 16000,
    "channels": 1,
    "length": 640
  }
}

{
  "kind": "AudioData",
  "audioData": {
    "timestamp": "2024-11-15T19:16:12.925Z",
    "participantRawID": "8:acs:3d20e1de-0f28-41c5…",
    "data": "5ADwAOMA6AD0A…",
    "silent": false
  }
}

var audioData = OutStreamingData.GetAudioDataForOutbound(audioData)),
byte[] jsonBytes = Encoding.UTF8.GetBytes(audioData);

// Write your logic to send the PCM audio chunk over the WebSocket
// Example of how to send audio data over the WebSocket
await m_webSocket.SendAsync(new ArraySegment < byte > (jsonBytes), WebSocketMessageType.Text, endOfMessage: true, CancellationToken.None);

var stopData = OutStreamingData.GetStopAudioForOutbound();
byte[] jsonBytes = Encoding.UTF8.GetBytes(stopData);

// Write your logic to send stop data to ACS over the WebSocket
// Example of how to send stop data over the WebSocket
await m_webSocket.SendAsync(new ArraySegment < byte > (jsonBytes), WebSocketMessageType.Text, endOfMessage: true, CancellationToken.None);

MediaStreamingOptions mediaStreamingOptions = new MediaStreamingOptions(appConfig.getTransportUrl(), MediaStreamingTransport.WEBSOCKET, MediaStreamingContent.AUDIO, MediaStreamingAudioChannel.MIXED, true).setEnableBidirectional(true).setAudioFormat(AudioFormat.PCM_24K_MONO);
options = new AnswerCallOptions(data.getString(INCOMING_CALL_CONTEXT), callbackUri).setCallIntelligenceOptions(callIntelligenceOptions).setMediaStreamingOptions(mediaStreamingOptions);
Response answerCallResponse = client.answerCallWithResponse(options, Context.NONE);

MediaStreamingOptions mediaStreamingOptions = new MediaStreamingOptions(MediaStreamingAudioChannel.UNMIXED);
mediaStreamingOptions.setTransportUrl(appConfig.getTransportUrl());
mediaStreamingOptions.setStartMediaStreaming(true);
mediaStreamingOptions.setEnableDtmfTones(true); // Allow receiving DTMF tones
mediaStreamingOptions.setEnableBidirectional(true);
mediaStreamingOptions.setAudioFormat(AudioFormat.PCM_24K_MONO);

options = new AnswerCallOptions(data.getString(INCOMING_CALL_CONTEXT), callbackUri)
    .setCallIntelligenceOptions(callIntelligenceOptions)
    .setMediaStreamingOptions(mediaStreamingOptions);

Response answerCallResponse = client.answerCallWithResponse(options, Context.NONE);

StartMediaStreamingOptions mediaStreamingOptions = new StartMediaStreamingOptions();
mediaStreamingOptions.setOperationContext("StartMediaStreamingContext");

client.getCallConnection(callConnectionId)
      .getCallMedia()
      .startMediaStreamingWithResponse(mediaStreamingOptions, Context.NONE);    

StopMediaStreamingOptions stopOptions = new StopMediaStreamingOptions();
stopOptions.setOperationContext("StopMediaStreamingContext");

client.getCallConnection(callConnectionId)
      .getCallMedia()
      .stopMediaStreamingWithResponse(stopOptions, Context.NONE);

@OnMessage
public void onMessage(String message, Session session) {
  System.out.println("Received message: " + message);
  var parsedData = StreamingData.parse(message);
  if (parsedData instanceof AudioData) {
    var audioData = (AudioData) parsedData;
    sendAudioData(session, audioData.getData());
  }
}

{
  "kind": "AudioMetadata",
  "audioMetadata": {
    "subscriptionId": "89e8cb59-b991-48b0-b154-1db84f16a077",
    "encoding": "PCM",
    "sampleRate": 16000,
    "channels": 1,
    "length": 640
  }
}

{
  "kind": "AudioData",
  "audioData": {
    "timestamp": "2024-11-15T19:16:12.925Z",
    "participantRawID": "8:acs:3d20e1de-0f28-41c5…",
    "data": "5ADwAOMA6AD0A…",
    "silent": false
  }
}

private void sendAudioData(Session session, byte[] binaryData) {
    System.out.println("Data buffer---> " + binaryData.getClass().getName());
    if (session.isOpen()) {
        try {
            var serializedData = OutStreamingData.getStreamingDataForOutbound(binaryData);
            session.getAsyncRemote().sendText(serializedData);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

private void stopAudio(Session session) {
    if (session.isOpen()) {
        try {
            var serializedData = OutStreamingData.getStopAudioForOutbound();
            session.getAsyncRemote().sendText(serializedData);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

const mediaStreamingOptions = {
    transportUrl: "wss://YOUR_WEBSOCKET_URL", 
    transportType: "websocket",               
    contentType: "audio",                     
    audioChannelType: "mixed",                
    startMediaStreaming: true,
    enableDtmfTones: true,           
    enableBidirectional: true,                
    audioFormat: "Pcm24KMono"                 
};

const answerCallOptions = {
    mediaStreamingOptions: mediaStreamingOptions
};

answerCallResult = await acsClient.answerCall(
    incomingCallContext,
    callbackUri,
    answerCallOptions
);

const mediaStreamingOptions: MediaStreamingOptions = {
	transportUrl:  "wss://YOUR_WEBSOCKET_URL",
	transportType: "websocket",
	contentType: "audio",
	audioChannelType: "unmixed",
	startMediaStreaming: false,
	enableDtmfTones: true,
	enableBidirectional: true,
	audioFormat: "Pcm24KMono"
}
const answerCallOptions: AnswerCallOptions = {
	mediaStreamingOptions: mediaStreamingOptions
};

answerCallResult = await acsClient.answerCall(
	incomingCallContext,
	callbackUri,
	answerCallOptions
);

const startMediaStreamingOptions: StartMediaStreamingOptions = {
	operationContext: "startMediaStreaming"
}

await answerCallResult.callConnection.getCallMedia().startMediaStreaming(startMediaStreamingOptions);

const stopMediaStreamingOptions: StopMediaStreamingOptions = {
	operationContext: "stopMediaStreaming"
}
await answerCallResult.callConnection.getCallMedia().stopMediaStreaming(stopMediaStreamingOptions);

wss.on('connection', async (ws: WebSocket) => {
	console.log('Client connected');
	await initWebsocket(ws);
	await startConversation();
	ws.on('message', async (packetData: ArrayBuffer) => {
		try {
			if (ws.readyState === WebSocket.OPEN) {
				await processWebsocketMessageAsync(packetData);
			} else {
				console.warn(`ReadyState: ${ws.readyState}`);
			}
		} catch (error) {
			console.error('Error processing WebSocket message:', error);
		}
	});
	ws.on('close', () => {
		console.log('Client disconnected');
	});
});

async function processWebsocketMessageAsync(receivedBuffer: ArrayBuffer) {
	const result = StreamingData.parse(receivedBuffer);
	const kind = StreamingData.getStreamingKind();

	// Get the streaming data kind  
	if (kind === StreamingDataKind.AudioData) {
		const audioData = (result as AudioData);
		// process your audio data  
	}
}

{
  "kind": "AudioMetadata",
  "audioMetadata": {
    "subscriptionId": "89e8cb59-b991-48b0-b154-1db84f16a077",
    "encoding": "PCM",
    "sampleRate": 16000,
    "channels": 1,
    "length": 640
  }
}

{
  "kind": "AudioData",
  "audioData": {
    "timestamp": "2024-11-15T19:16:12.925Z",
    "participantRawID": "8:acs:3d20e1de-0f28-41c5…",
    "data": "5ADwAOMA6AD0A…",
    "silent": false
  }
}

async function receiveAudioForOutbound(data: string) { 
    try {
        const jsonData = OutStreamingData.getStreamingDataForOutbound(data);
        if (ws.readyState === WebSocket.OPEN) {
            ws.send(jsonData);
        } else {
            console.log("socket connection is not open.");
        }
    } catch (e) {
        console.log(e);
    }
}

async function stopAudio() {
	try {
		const jsonData = OutStreamingData.getStopAudioForOutbound();
		if (ws.readyState === WebSocket.OPEN) {
			ws.send(jsonData);
		} else {
			console.log("socket connection is not open.");
		}
	} catch (e) {
		console.log(e);
	}
}

media_streaming_options = MediaStreamingOptions(
    transport_url=WEBSOCKET_URI_HOST,
    transport_type=StreamingTransportType.WEBSOCKET,
    content_type=MediaStreamingContentType.AUDIO,
    audio_channel_type=MediaStreamingAudioChannelType.MIXED,
    start_media_streaming=True,
    enable_bidirectional=True,
    enable_dtmf_tones=True,
    audio_format=AudioFormat.PCM24_K_MONO
)
answer_call_result = call_automation_client.answer_call(
    incoming_call_context=incoming_call_context,
    media_streaming=media_streaming_configuration,
    callback_url=callback_uri,
)

media_streaming_options = MediaStreamingOptions(
    transport_url=WEBSOCKET_URI_HOST,
    transport_type=StreamingTransportType.WEBSOCKET,
    content_type=MediaStreamingContentType.AUDIO,
    audio_channel_type=MediaStreamingAudioChannelType.MIXED,
    start_media_streaming=True,
    enable_bidirectional=True,
    enable_dtmf_tones=True,
    audio_format=AudioFormat.PCM24_K_MONO
)

answer_call_result = call_automation_client.answer_call(
    incoming_call_context=incoming_call_context,
    media_streaming=media_streaming_configuration,
    callback_url=callback_uri
)

call_automation_client.get_call_connection(call_connection_id).start_media_streaming(
    operation_context="startMediaStreamingContext"
)

call_automation_client.get_call_connection(call_connection_id).stop_media_streaming(operation_context="stopMediaStreamingContext") 

async def handle_client(websocket):
    print("Client connected")
    try:
        async for message in websocket:
            json_object = json.loads(message)
            kind = json_object["kind"]
            if kind == "AudioData":
                audio_data = json_object["audioData"]["data"]
                # process your audio data
    except websockets.exceptions.ConnectionClosedOK:
        print("Client disconnected")
    except websockets.exceptions.ConnectionClosedError as e:
        print(f"Connection closed with error: {e}")
    except Exception as e:
        print(f"Unexpected error: {e}")

{
  "kind": "AudioMetadata",
  "audioMetadata": {
    "subscriptionId": "89e8cb59-b991-48b0-b154-1db84f16a077",
    "encoding": "PCM",
    "sampleRate": 16000,
    "channels": 1,
    "length": 640
  }
}

{
  "kind": "AudioData",
  "audioData": {
    "timestamp": "2024-11-15T19:16:12.925Z",
    "participantRawID": "8:acs:3d20e1de-0f28-41c5…",
    "data": "5ADwAOMA6AD0A…",
    "silent": false
  }
}

async def send_data(websocket, buffer):
    if websocket.open:
        data = {
            "Kind": "AudioData",
            "AudioData": {
                "Data": buffer
            },
            "StopAudio": None
        }
        # Serialize the server streaming data
        serialized_data = json.dumps(data)
        print(f"Out Streaming Data ---> {serialized_data}")
        # Send the chunk over the WebSocket
        await websocket.send(serialized_data)

async def stop_audio(websocket):
    if websocket.open:
        data = {
            "Kind": "StopAudio",
            "AudioData": None,
            "StopAudio": {}
        }
        # Serialize the server streaming data
        serialized_data = json.dumps(data)
        print(f"Out Streaming Data ---> {serialized_data}")
        # Send the chunk over the WebSocket
        await websocket.send(serialized_data)

Очистка ресурсов

Если вы хотите отменить и удалить подписку на Службы коммуникации, можно удалить ресурс или группу ресурсов. При удалении группы ресурсов также удаляются все связанные с ней ресурсы. См. сведения об очистке ресурсов.

Следующие шаги

• Узнать больше о потоковой передаче аудио.
• Дополнительные сведения об автоматизации вызовов и ее функциях.
• Узнайте больше об игровом действии .
• Узнайте больше о действии Распознавания.