1. 下載插件：motionverse官網(wǎng)地址：概述?·?Motionverse?接口文檔?(deepscience.cn)
2. 按照官方文檔新建Unity工程：對接說明?·?Motionverse?接口文檔?(deepscience.cn)
3. 通過我們自己的ASR，將語音轉(zhuǎn)換為文本，文本通過大語言模型（chatgpt、文心一言以及其他大語言模型）生成的結(jié)果，直接通過生成式AI技術(shù)，把文本轉(zhuǎn)化為AI智能體的聲音、動作和表情，和大語言模型完美連接，只需要在獲取到文本的時候調(diào)用以下代碼即可：
代碼示例：
DriveTask task = new DriveTask();
task.player = player;
task.text = question;
NLPDrive.GetDrive(task);
其中，player即為掛載motionverse插件中Player腳本的對象，question即為大語言模型獲取到的答案。
還可以自己生成語音，通過語音鏈接調(diào)用以下代碼：
DriveTask task = new DriveTask();
task.player = player;
task.text = audioUrl;
AudioUrlDrive.GetDrive(task);其中，player即為掛載motionverse插件中Player腳本的對象，audioUrl即為語音鏈接。
4. 新建腳本AskManager，并掛載到場景中（可新建空物體），腳本代碼如下：
using LitJson;
using Motionverse;
using MotionverseSDK;
using System;
using System.Collections;
using System.Text;
using UnityEngine;
using UnityEngine.Networking;
using UnityEngine.UI;
public class AskManager : MonoBehaviour
{
public Player player;
public Dropdown dropdown;
public InputField inputField;
public Button btnSend;
public string chatGptKey = "";
private string chatUrl = "https://chatgpt.kazava.io/v1/chat/completions";
public string wenXinAPI = "";
public string wenXinSECRET = "";
private string wenXinToken = "";
private int curSelectNLP = 0;
// Start is called before the first frame update
private void Awake()
{
for (int i = 0; i < Display.displays.Length; i++)
{
Display.displays[i].Activate();
}
}
void Start()
{
dropdown.onValueChanged.AddListener((value) =>
{
curSelectNLP = value;
});
StartCoroutine(GetWenxinToken());
btnSend.onClick.AddListener(() =>
{
if (string.IsNullOrEmpty(inputField.text))
{
Debug.Log("請輸入內(nèi)容！");
}
else
{
GetAnswer(inputField.text);
}
});
}
public void GetAnswer(string q)
{
StartCoroutine(RealAnswer(q));
}
public IEnumerator RealAnswer(string question)
{
switch (curSelectNLP)
{
case 0:
DriveTask task = new DriveTask();
task.player = player;
task.text = question;
NLPDrive.GetDrive(task);
break;
case 1:
StartCoroutine(RequestChat(question));
break;
case 2:
StartCoroutine(ChatCompletions(question));
break;
default:
break;
}
Invoke("restartRecording", 1);
yield return null;
}
IEnumerator GetWenxinToken()
{
string url =$"https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id={wenXinAPI}&client_secret={wenXinSECRET}";
UnityWebRequest webRequest = UnityWebRequest.Get(url);
webRequest.timeout = 5000;
yield return webRequest.SendWebRequest();
if (webRequest.result == UnityWebRequest.Result.Success)
{
string response = webRequest.downloadHandler.text;
var result = JsonUtility.FromJson<AccessTokenResponse>(response);
wenXinToken = result.access_token;
}
else
{
Debug.LogError("Failed to get access token: " + webRequest.error);
}
}
IEnumerator ChatCompletions(string content)
{
string url = $"https://aip.baidubce.com/rpc/2.0/ai_custom/v1/wenxinworkshop/chat/completions?access_token={wenXinToken}";
WenXinPostData postData = new WenXinPostData();
postData.messages = new WenXinMessage[1];
postData.messages[0] = new WenXinMessage();
postData.messages[0].content = content + "30字以內(nèi)";
string data = JsonMapper.ToJson(postData);
UnityWebRequest webRequest = UnityWebRequest.Post(url, data, "application/json");
webRequest.timeout = 5000;
yield return webRequest.SendWebRequest();
if (webRequest.result == UnityWebRequest.Result.Success)
{
string response = webRequest.downloadHandler.text;
WenXinRequest requestData = JsonMapper.ToObject<WenXinRequest>(response);
DriveTask task = new DriveTask();
task.player = player;
task.text = requestData.result;
TextDrive.GetDrive(task);
}
else
{
Debug.LogError("Chat completions request failed: " + webRequest.error);
}
}
private IEnumerator RequestChat(string content)
{
using (UnityWebRequest webRequest = new UnityWebRequest(chatUrl, "POST"))
{
webRequest.SetRequestHeader("Content-Type", "application/json");
ChatGPTPostData postData = new ChatGPTPostData();
postData.key = chatGptKey;
postData.messages = new PostMessage[1];
postData.messages[0] = new PostMessage();
postData.messages[0].content = content + "30字以內(nèi)"; ;
string data = JsonMapper.ToJson(postData);
byte[] jsonToSend = new UTF8Encoding().GetBytes(data);
webRequest.uploadHandler = new UploadHandlerRaw(jsonToSend);
webRequest.downloadHandler = new DownloadHandlerBuffer();
yield return webRequest.SendWebRequest();
if (webRequest.result != UnityWebRequest.Result.Success)
{
Debug.LogError("ChatGPT request error: " + content + webRequest.error);
}
else
{
string response = webRequest.downloadHandler.text;
ChatGPTRequestData requestData = JsonMapper.ToObject<ChatGPTRequestData>(response);
DriveTask task = new DriveTask();
task.player = player;
task.text = requestData.choices[0].message.content;
TextDrive.GetDrive(task);
}
}
}
}
5. 新建腳本RealtimeAsrManager，代碼如下：
using System;
using System.Collections;
using UnityEngine;
using UnityEngine.UI;
using UnityWebSocket;
namespace Motionverse
{
public class RealtimeAsrManager : MonoBehaviour
{
private IWebSocket webSocket;
private Status status = Status.FirstFrame;
private bool lockReconnect = false;
[HideInInspector]
public static Action<string> Asr;
[SerializeField]
private GameObject TextBG;
//百度
public int AsrAppId;
public string AsrAppkey = "";
void Start()
{
CreateWebSocket();
RecorderManager.DataAvailable += OnDataAvailable;
}
private void OnDisable()
{
RecorderManager.DataAvailable -= OnDataAvailable;
}
void CreateWebSocket()
{
try
{
string uril = "wss://vop.baidu.com/realtime_asr?sn=" + Guid.NewGuid().ToString();
webSocket = new WebSocket(uril);
InitHandle();
webSocket.ConnectAsync();
}
catch (Exception e)
{
Debug.Log("websocket連接異常:" + e.Message);
ReConnect();
}
}
private void InitHandle()
{
RemoveHandle();
webSocket.OnOpen += OnOpen;
webSocket.OnMessage += OnMessage;
webSocket.OnClose += OnClose;
webSocket.OnError += OnError;
}
void RemoveHandle()
{
webSocket.OnOpen -= OnOpen;
webSocket.OnMessage -= OnMessage;
webSocket.OnClose -= OnClose;
webSocket.OnError -= OnError;
}
//請求開始
private void OnDataAvailable(byte[] data)
{
if (webSocket == null || (webSocket != null && webSocket.ReadyState != WebSocketState.Open))
return;
switch (status)
{
case Status.FirstFrame://握手
{
var firstFrame = new FirstFrame();
firstFrame.data.appid= AsrAppId;
firstFrame.data.appkey = AsrAppkey;
webSocket.SendAsync(JsonUtility.ToJson(firstFrame));
status = Status.ContinueFrame;
}
break;
case Status.ContinueFrame://開始發(fā)送
{
if (data.Length > 0)
{
webSocket.SendAsync(data);
}
}
break;
case Status.LastFrame://關(guān)閉
{
webSocket.SendAsync(JsonUtility.ToJson(new LastFrame()));
}
break;
default:
break;
}
}
void ReConnect()
{
if (this.lockReconnect)
return;
this.lockReconnect = true;
StartCoroutine(SetReConnect());
}
private IEnumerator SetReConnect()
{
yield return new WaitForSeconds(1);
CreateWebSocket();
lockReconnect = false;
}
#region WebSocket Event Handlers
private void OnOpen(object sender, OpenEventArgs e)
{
status = Status.FirstFrame;
}
private void OnMessage(object sender, MessageEventArgs e)
{
var err_msg = Utils.GetJsonValue(e.Data, "err_msg");
var type = Utils.GetJsonValue(e.Data, "type");
if (err_msg == "OK" && type == "MID_TEXT")
{
var result = Utils.GetJsonValue(e.Data, "result");
TextBG.GetComponentInChildren<Text>().text = result;
}
if (err_msg == "OK" && type == "FIN_TEXT")
{
var result = Utils.GetJsonValue(e.Data, "result");
TextBG.GetComponentInChildren<Text>().text = result;
if (result.Length > 1)
{
RecorderManager.Instance.EndRecording();
Asr?.Invoke(result);
}
}
}
private void OnClose(object sender, CloseEventArgs e)
{
Debug.Log("websocket關(guān)閉," + string.Format("Closed: StatusCode: {0}, Reason: {1}", e.StatusCode, e.Reason));
webSocket = null;
ReConnect();
}
private void OnError(object sender, ErrorEventArgs e)
{
if (e != null)
Debug.Log("websocket連接異常:" + e.Message);
webSocket = null;
ReConnect();
}
#endregion
}
}
6. 新建腳本RecorderManager，代碼如下：
using UnityEngine;
using System;
using System.Collections;
using UnityEngine.UI;
using Unity.VisualScripting;
namespace Motionverse
{
public class RecorderManager : Singleton<RecorderManager>
{
//標(biāo)記是否有麥克風(fēng)
private bool isHaveMic = false;
//當(dāng)前錄音設(shè)備名稱
private string currentDeviceName = string.Empty;
//表示錄音的最大時長
int recordMaxLength = 600;
//錄音頻率,控制錄音質(zhì)量(16000)
int recordFrequency = 16000;
[HideInInspector]
public static Action<byte[]> DataAvailable;
[SerializeField]
private GameObject micON;
[SerializeField]
private GameObject micOFF;
[SerializeField]
private Image micAmount;
[SerializeField]
private GameObject TextBG;
private AudioClip saveAudioClip;
int offsetSamples = 0;
private void Start()
{
Debug.Log(Microphone.devices[0]);
if (Microphone.devices.Length > 0)
{
isHaveMic = true;
currentDeviceName = Microphone.devices[0];
StartCoroutine(GetAudioFrames());
StartRecording();
}
}
/// <summary>
/// 開始錄音
/// </summary>
/// <returns></returns>
public void StartRecording() //16000
{
if (isHaveMic == false || Microphone.IsRecording(currentDeviceName))
{
return;
}
micOFF.gameObject.SetActive(false);
micON.gameObject.SetActive(true);
TextBG.GetComponentInChildren<Text>().text = null;
offsetSamples = 0;
saveAudioClip = Microphone.Start(currentDeviceName, true, recordMaxLength, recordFrequency);
}
public void OnButtonClick()
{
if (Microphone.IsRecording(currentDeviceName))
{
EndRecording();
}
else
{
StartRecording();
}
}
public void EndRecording()
{
if (isHaveMic == false || !Microphone.IsRecording(currentDeviceName))
{
return;
}
micOFF.gameObject.SetActive(true);
micON.gameObject.SetActive(false);
//結(jié)束錄音
Microphone.End(currentDeviceName);
}
public bool IsRecording()
{
return Microphone.IsRecording(currentDeviceName);
}
IEnumerator GetAudioFrames()
{
while (true)
{
if (Microphone.IsRecording(currentDeviceName))
{
int lenght = Microphone.GetPosition(currentDeviceName) * saveAudioClip.channels - offsetSamples;
if (lenght > 0)
{
float[] samples = new float[lenght];
saveAudioClip.GetData(samples, offsetSamples);
var samplesShort = new short[samples.Length];
for (var index = 0; index < samples.Length; index++)
{
samplesShort[index] = (short)(samples[index] * short.MaxValue);
}
byte[] binaryData = new byte[samplesShort.Length * 2];
Buffer.BlockCopy(samplesShort, 0, binaryData, 0, binaryData.Length);
offsetSamples += lenght;
DataAvailable?.Invoke(binaryData);
}
yield return new WaitForSeconds(0.16f);
}
else
{
yield return new WaitForSeconds(0.16f);
byte[] binaryData = new byte[2];
DataAvailable?.Invoke(binaryData);
}
}
}
/// <summary>
/// 獲取毫秒級別的時間戳,用于計算按下錄音時長
/// </summary>
/// <returns></returns>
public double GetTimestampOfNowWithMillisecond()
{
return (DateTime.Now.ToUniversalTime().Ticks - 621355968000000000) / 10000;
}
private void LateUpdate()
{
if (isHaveMic == true && Microphone.IsRecording(currentDeviceName))
{
micAmount.fillAmount = Volume;
}
}
public float Volume
{
get
{
if (Microphone.IsRecording(currentDeviceName))
{
// 采樣數(shù)
int sampleSize = 128;
float[] samples = new float[sampleSize];
int startPosition = Microphone.GetPosition(currentDeviceName) - (sampleSize + 1);
// 得到數(shù)據(jù)
if (startPosition < 0)
return 0;
saveAudioClip.GetData(samples, startPosition);
// Getting a peak on the last 128 samples
float levelMax = 0;
for (int i = 0; i < sampleSize; ++i)
{
float wavePeak = samples[i];
if (levelMax < wavePeak)
levelMax = wavePeak;
}
return levelMax;
}
return 0;
}
}
void OnGUI()
{
GUIStyle guiStyle = GUIStyle.none;
guiStyle.fontSize = 10;
guiStyle.normal.textColor = Color.white;
guiStyle.alignment = TextAnchor.UpperLeft;
Rect tr = new Rect(0, 0, 100, 100);
GUI.Label(tr, currentDeviceName, guiStyle);
}
}
}
7. 新建空物體，掛載腳本RecorderManager和RealtimeAsrManager
8. 輸入百度asr的appId和secretKey：
9. 輸入GPTkey、問心一眼appId和secretKey：
10. 根據(jù)需求選擇相應(yīng)的NLP方式：
注意：如果缺少Singleton文件，可使用如下代碼：文章來源：http://www.zghlxwxcb.cn/news/detail-694401.html
using UnityEngine;
namespace Motionverse
{
public abstract class Singleton<T> : MonoBehaviour where T : Singleton<T>
{
private static T sInstance = null;
public static T Instance
{
get
{
if (sInstance == null)
{
GameObject gameObject = new(typeof(T).FullName);
sInstance = gameObject.AddComponent<T>();
}
return sInstance;
}
}
public static void Clear()
{
sInstance = null;
}
protected virtual void Awake()
{
if (sInstance != null) Debug.LogError(name + "error: already initialized", this);
sInstance = (T)this;
}
}
}若有收獲，就點個贊吧文章來源地址http://www.zghlxwxcb.cn/news/detail-694401.html
到了這里，關(guān)于數(shù)字人系列四：Motionverse 接入chatgpt、文心一言等國內(nèi)外大語言模型的文章就介紹完了。如果您還想了解更多內(nèi)容，請在右上角搜索TOY模板網(wǎng)以前的文章或繼續(xù)瀏覽下面的相關(guān)文章，希望大家以后多多支持TOY模板網(wǎng)！
