xiaohei/digital_person
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ai state machine init

Browse Source
This commit is contained in:
gcw_4spBpAfv
2026-03-05 18:50:48 +08:00
parent ef2bada800
commit d5767156b9
10 changed files with 855 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

373
app/src/main/java/com/digitalperson/Live2DChatActivity.kt
View File

@@ -27,10 +27,15 @@ import com.digitalperson.face.ImageProxyBitmapConverter
import com.digitalperson.metrics.TraceManager
import com.digitalperson.metrics.TraceSession
import com.digitalperson.tts.TtsController
import com.digitalperson.interaction.DigitalHumanInteractionController
import com.digitalperson.interaction.InteractionActionHandler
import com.digitalperson.interaction.InteractionState
import com.digitalperson.interaction.UserMemoryStore
import com.digitalperson.llm.LLMManager
import com.digitalperson.llm.LLMManagerCallback
import com.digitalperson.util.FileHelper
import java.io.File
import org.json.JSONObject
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import kotlinx.coroutines.CoroutineScope
@@ -85,9 +90,18 @@ class Live2DChatActivity : AppCompatActivity() {
private lateinit var cameraPreviewView: PreviewView
private lateinit var faceOverlayView: FaceOverlayView
private lateinit var faceDetectionPipeline: FaceDetectionPipeline
private lateinit var interactionController: DigitalHumanInteractionController
private lateinit var userMemoryStore: UserMemoryStore
private var facePipelineReady: Boolean = false
private var cameraProvider: ProcessCameraProvider? = null
private lateinit var cameraAnalyzerExecutor: ExecutorService
private var activeUserId: String = "guest"
private var pendingLocalThoughtCallback: ((String) -> Unit)? = null
private var pendingLocalProfileCallback: ((String) -> Unit)? = null
private var localThoughtSilentMode: Boolean = false
private val recentConversationLines = ArrayList<String>()
private var recentConversationDirty: Boolean = false
private var lastFacePresent: Boolean = false
override fun onRequestPermissionsResult(
requestCode: Int,
@@ -143,15 +157,73 @@ class Live2DChatActivity : AppCompatActivity() {
cameraPreviewView.implementationMode = PreviewView.ImplementationMode.COMPATIBLE
faceOverlayView = findViewById(R.id.face_overlay)
cameraAnalyzerExecutor = Executors.newSingleThreadExecutor()
userMemoryStore = UserMemoryStore(applicationContext)
interactionController = DigitalHumanInteractionController(
scope = ioScope,
handler = object : InteractionActionHandler {
override fun onStateChanged(state: InteractionState) {
runOnUiThread {
uiManager.appendToUi("\n[State] $state\n")
}
if (state == InteractionState.IDLE) {
analyzeUserProfileInIdleIfNeeded()
}
}
override fun playMotion(motionName: String) {
playInteractionMotion(motionName)
}
override fun appendText(text: String) {
uiManager.appendToUi(text)
}
override fun speak(text: String) {
ttsController.enqueueSegment(text)
ttsController.enqueueEnd()
}
override fun requestCloudReply(userText: String) {
llmInFlight = true
Log.i(TAG_LLM, "Routing dialogue to CLOUD")
cloudApiManager.callLLM(buildCloudPromptWithUserProfile(userText))
}
override fun requestLocalThought(prompt: String, onResult: (String) -> Unit) {
this@Live2DChatActivity.requestLocalThought(prompt, onResult)
}
override fun onRememberUser(faceIdentityId: String, name: String?) {
activeUserId = faceIdentityId
userMemoryStore.upsertUserSeen(activeUserId, name)
}
override fun saveThought(thought: String) {
userMemoryStore.upsertUserSeen(activeUserId, null)
userMemoryStore.updateThought(activeUserId, thought)
}
override fun loadLatestThought(): String? = userMemoryStore.getLatestThought()
override fun addToChatHistory(role: String, content: String) {
appendConversationLine(role, content)
}
override fun addAssistantMessageToCloudHistory(content: String) {
cloudApiManager.addAssistantMessage(content)
}
}
)
faceDetectionPipeline = FaceDetectionPipeline(
context = applicationContext,
onResult = { result ->
faceOverlayView.updateResult(result)
},
onGreeting = { greeting ->
uiManager.appendToUi("\n[Face] $greeting\n")
ttsController.enqueueSegment(greeting)
ttsController.enqueueEnd()
onPresenceChanged = { present, faceIdentityId, recognizedName ->
if (present == lastFacePresent) return@FaceDetectionPipeline
lastFacePresent = present
Log.d(TAG_ACTIVITY, "present=$present, faceIdentityId=$faceIdentityId, recognized=$recognizedName")
interactionController.onFacePresenceChanged(present, faceIdentityId, recognizedName)
}
)
@@ -200,13 +272,12 @@ class Live2DChatActivity : AppCompatActivity() {
// 设置 LLM 模式开关
uiManager.setLLMSwitchListener { isChecked ->
// 交互状态机固定路由：用户对话走云端，回忆走本地。此开关仅作为本地LLM可用性提示。
useLocalLLM = isChecked
Log.i(TAG_LLM, "LLM mode switched: useLocalLLM=$useLocalLLM")
uiManager.showToast("LLM模式已切换到${if (isChecked) "本地" else "云端"}")
// 重新初始化 LLM
initLLM()
uiManager.showToast("状态机路由已固定：对话云端，回忆本地")
}
// 默认不显示 LLM 开关，等模型下载完成后再显示
uiManager.showLLMSwitch(false)
if (AppConfig.USE_HOLD_TO_SPEAK) {
uiManager.setButtonsEnabled(recordEnabled = false)
@@ -226,8 +297,9 @@ class Live2DChatActivity : AppCompatActivity() {
vadManager = VadManager(this)
vadManager.setCallback(createVadCallback())
// 初始化 LLM 管理器
// 初始化本地 LLM（用于 memory 状态）
initLLM()
interactionController.start()
// 检查是否需要下载模型
if (!FileHelper.isLocalLLMAvailable(this)) {
@@ -259,8 +331,8 @@ class Live2DChatActivity : AppCompatActivity() {
if (FileHelper.isLocalLLMAvailable(this)) {
Log.i(AppConfig.TAG, "Local LLM is available, enabling local LLM switch")
// 显示本地 LLM 开关，并同步状态
uiManager.showLLMSwitch(true)
uiManager.setLLMSwitchChecked(useLocalLLM)
uiManager.showLLMSwitch(false)
initLLM()
}
} else {
Log.e(AppConfig.TAG, "Failed to download model files: $message")
@@ -275,8 +347,7 @@ class Live2DChatActivity : AppCompatActivity() {
// 模型已存在，直接初始化其他组件
initializeOtherComponents()
// 显示本地 LLM 开关，并同步状态
uiManager.showLLMSwitch(true)
uiManager.setLLMSwitchChecked(useLocalLLM)
uiManager.showLLMSwitch(false)
}
}
@@ -348,6 +419,7 @@ class Live2DChatActivity : AppCompatActivity() {
runOnUiThread {
uiManager.appendToUi("\n\n[ASR] ${text}\n")
}
appendConversationLine("用户", text)
currentTrace?.markRecordingDone()
currentTrace?.markLlmResponseReceived()
}
@@ -361,17 +433,8 @@ class Live2DChatActivity : AppCompatActivity() {
override fun isLlmInFlight(): Boolean = llmInFlight
override fun onLlmCalled(text: String) {
llmInFlight = true
Log.d(AppConfig.TAG, "Calling LLM with text: $text")
if (useLocalLLM) {
Log.i(TAG_LLM, "Routing to LOCAL LLM")
// 使用本地 LLM 生成回复
generateResponse(text)
} else {
Log.i(TAG_LLM, "Routing to CLOUD LLM")
// 使用云端 LLM 生成回复
cloudApiManager.callLLM(text)
}
Log.d(AppConfig.TAG, "Forward ASR text to interaction controller: $text")
interactionController.onUserAsrText(text)
}
}
@@ -390,6 +453,7 @@ class Live2DChatActivity : AppCompatActivity() {
override fun onLLMResponseReceived(response: String) {
currentTrace?.markLlmDone()
llmInFlight = false
appendConversationLine("助手", response)
if (enableStreaming) {
for (seg in segmenter.flush()) {
@@ -411,6 +475,7 @@ class Live2DChatActivity : AppCompatActivity() {
ttsController.enqueueSegment(filteredText)
ttsController.enqueueEnd()
}
interactionController.onDialogueResponseFinished()
}
override fun onLLMStreamingChunkReceived(chunk: String) {
@@ -442,6 +507,7 @@ class Live2DChatActivity : AppCompatActivity() {
override fun onError(errorMessage: String) {
llmInFlight = false
uiManager.showToast(errorMessage, Toast.LENGTH_LONG)
interactionController.onDialogueResponseFinished()
onStopClicked(userInitiated = false)
}
}
@@ -479,6 +545,7 @@ class Live2DChatActivity : AppCompatActivity() {
override fun onDestroy() {
super.onDestroy()
try { interactionController.stop() } catch (_: Throwable) {}
stopCameraPreviewAndDetection()
onStopClicked(userInitiated = false)
ioScope.cancel()
@@ -490,6 +557,7 @@ class Live2DChatActivity : AppCompatActivity() {
try { cameraAnalyzerExecutor.shutdown() } catch (_: Throwable) {}
try { ttsController.release() } catch (_: Throwable) {}
try { llmManager?.destroy() } catch (_: Throwable) {}
try { userMemoryStore.close() } catch (_: Throwable) {}
try { uiManager.release() } catch (_: Throwable) {}
try { audioProcessor.release() } catch (_: Throwable) {}
}
@@ -757,85 +825,158 @@ class Live2DChatActivity : AppCompatActivity() {
}
Log.d(AppConfig.TAG, "processSamplesLoop stopped")
}
private fun playInteractionMotion(motionName: String) {
when (motionName) {
"haru_g_m22.motion3.json" -> uiManager.setMood("高兴")
"haru_g_m01.motion3.json", "haru_g_m17.motion3.json" -> uiManager.setMood("中性")
"haru_g_m15.motion3.json" -> uiManager.setMood("关心")
"haru_g_idle.motion3.json" -> uiManager.setMood("平和")
else -> uiManager.setMood("中性")
}
}
private fun appendConversationLine(role: String, text: String) {
val line = "$role: ${text.trim()}"
if (line.length <= 4) return
recentConversationLines.add(line)
if (recentConversationLines.size > 12) {
recentConversationLines.removeAt(0)
}
recentConversationDirty = true
}
private fun buildCloudPromptWithUserProfile(userText: String): String {
val profile = userMemoryStore.getMemory(activeUserId) ?: return userText
val profileParts = ArrayList<String>()
profile.displayName?.takeIf { it.isNotBlank() }?.let { profileParts.add("姓名:$it") }
profile.age?.takeIf { it.isNotBlank() }?.let { profileParts.add("年龄:$it") }
profile.gender?.takeIf { it.isNotBlank() }?.let { profileParts.add("性别:$it") }
profile.hobbies?.takeIf { it.isNotBlank() }?.let { profileParts.add("爱好:$it") }
profile.profileSummary?.takeIf { it.isNotBlank() }?.let { profileParts.add("画像:$it") }
if (profileParts.isEmpty()) return userText
return buildString {
append("[用户画像]\n")
append(profileParts.joinToString("；"))
append("\n[/用户画像]\n")
append(userText)
}
}
private fun analyzeUserProfileInIdleIfNeeded() {
if (!recentConversationDirty || !activeUserId.startsWith("face_")) return
if (recentConversationLines.isEmpty()) return
val dialogue = recentConversationLines.joinToString("\n")
requestLocalProfileExtraction(dialogue) { raw ->
try {
val json = parseFirstJsonObject(raw)
val name = json.optString("name", "").trim().ifBlank { null }
val age = json.optString("age", "").trim().ifBlank { null }
val gender = json.optString("gender", "").trim().ifBlank { null }
val hobbies = json.optString("hobbies", "").trim().ifBlank { null }
val summary = json.optString("summary", "").trim().ifBlank { null }
if (name != null) {
userMemoryStore.updateDisplayName(activeUserId, name)
}
userMemoryStore.updateProfile(activeUserId, age, gender, hobbies, summary)
recentConversationDirty = false
runOnUiThread {
uiManager.appendToUi("\n[Memory] 已更新用户画像: $activeUserId\n")
}
} catch (e: Exception) {
Log.w(TAG_LLM, "Profile parse failed: ${e.message}")
}
}
}
private fun requestLocalProfileExtraction(dialogue: String, onResult: (String) -> Unit) {
try {
val local = llmManager
if (local == null) {
onResult("{}")
return
}
localThoughtSilentMode = true
pendingLocalProfileCallback = onResult
Log.i(TAG_LLM, "Routing profile extraction to LOCAL")
local.generateResponseWithSystem(
"你是信息抽取器。仅输出JSON对象，不要其他文字。字段为name,age,gender,hobbies,summary。",
"请从以下对话提取用户信息，未知填空字符串：\n$dialogue"
)
} catch (e: Exception) {
pendingLocalProfileCallback = null
localThoughtSilentMode = false
Log.e(TAG_LLM, "requestLocalProfileExtraction failed: ${e.message}", e)
onResult("{}")
}
}
private fun parseFirstJsonObject(text: String): JSONObject {
val raw = text.trim()
val start = raw.indexOf('{')
val end = raw.lastIndexOf('}')
if (start >= 0 && end > start) {
return JSONObject(raw.substring(start, end + 1))
}
return JSONObject(raw)
}
/**
* 初始化 LLM 管理器
* 初始化本地 LLM（仅用于回忆状态）
*/
private fun initLLM() {
try {
Log.i(TAG_LLM, "initLLM called, useLocalLLM=$useLocalLLM")
Log.i(TAG_LLM, "initLLM called for memory-local model")
llmManager?.destroy()
llmManager = null
if (useLocalLLM) {
// // 本地 LLM 初始化前，先暂停/释放重模块
// Log.i(AppConfig.TAG, "Pausing camera and releasing face detection before LLM initialization")
// stopCameraPreviewAndDetection()
// try {
// faceDetectionPipeline.release()
// Log.i(AppConfig.TAG, "Face detection pipeline released")
// } catch (e: Exception) {
// Log.w(AppConfig.TAG, "Failed to release face detection pipeline: ${e.message}")
// }
// // 释放 VAD 管理器
// try {
// vadManager.release()
// Log.i(AppConfig.TAG, "VAD manager released")
// } catch (e: Exception) {
// Log.w(AppConfig.TAG, "Failed to release VAD manager: ${e.message}")
// }
val modelPath = FileHelper.getLLMModelPath(applicationContext)
if (!File(modelPath).exists()) {
throw IllegalStateException("RKLLM model file missing: $modelPath")
}
Log.i(AppConfig.TAG, "Initializing LLM with model path: $modelPath")
val localLlmResponseBuffer = StringBuilder()
llmManager = LLMManager(modelPath, object : LLMManagerCallback {
override fun onThinking(msg: String, finished: Boolean) {
// 处理思考过程
Log.d(TAG_LLM, "LOCAL onThinking finished=$finished msg=${msg.take(60)}")
runOnUiThread {
if (!finished && enableStreaming) {
uiManager.appendToUi("\n[LLM] 思考中: $msg\n")
}
}
}
override fun onResult(msg: String, finished: Boolean) {
// 处理生成结果
Log.d(TAG_LLM, "LOCAL onResult finished=$finished len=${msg.length}")
runOnUiThread {
if (!finished) {
localLlmResponseBuffer.append(msg)
if (enableStreaming) {
uiManager.appendToUi(msg)
}
} else {
val finalText = localLlmResponseBuffer.toString().trim()
localLlmResponseBuffer.setLength(0)
if (!enableStreaming && finalText.isNotEmpty()) {
uiManager.appendToUi("$finalText\n")
}
uiManager.appendToUi("\n\n[LLM] 生成完成\n")
llmInFlight = false
if (finalText.isNotEmpty()) {
ttsController.enqueueSegment(finalText)
ttsController.enqueueEnd()
} else {
Log.w(TAG_LLM, "LOCAL final text is empty, skip TTS enqueue")
}
}
}
}
})
Log.i(AppConfig.TAG, "LLM initialized successfully")
Log.i(TAG_LLM, "LOCAL LLM initialized")
} else {
// 使用云端 LLM，不需要初始化本地 LLM
Log.i(AppConfig.TAG, "Using cloud LLM, skipping local LLM initialization")
Log.i(TAG_LLM, "CLOUD mode active")
val modelPath = FileHelper.getLLMModelPath(applicationContext)
if (!File(modelPath).exists()) {
throw IllegalStateException("RKLLM model file missing: $modelPath")
}
Log.i(AppConfig.TAG, "Initializing local memory LLM with model path: $modelPath")
val localLlmResponseBuffer = StringBuilder()
llmManager = LLMManager(modelPath, object : LLMManagerCallback {
override fun onThinking(msg: String, finished: Boolean) {
Log.d(TAG_LLM, "LOCAL onThinking finished=$finished msg=${msg.take(60)}")
}
override fun onResult(msg: String, finished: Boolean) {
Log.d(TAG_LLM, "LOCAL onResult finished=$finished len=${msg.length}")
runOnUiThread {
if (!finished) {
localLlmResponseBuffer.append(msg)
if (enableStreaming && !localThoughtSilentMode) {
uiManager.appendToUi(msg)
}
return@runOnUiThread
}
val finalText = localLlmResponseBuffer.toString().trim()
localLlmResponseBuffer.setLength(0)
val profileCallback = pendingLocalProfileCallback
pendingLocalProfileCallback = null
if (profileCallback != null) {
profileCallback(finalText)
localThoughtSilentMode = false
return@runOnUiThread
}
val callback = pendingLocalThoughtCallback
pendingLocalThoughtCallback = null
if (callback != null) {
callback(finalText)
localThoughtSilentMode = false
return@runOnUiThread
}
if (!localThoughtSilentMode && finalText.isNotEmpty()) {
uiManager.appendToUi("$finalText\n")
ttsController.enqueueSegment(finalText)
ttsController.enqueueEnd()
}
localThoughtSilentMode = false
}
}
})
Log.i(TAG_LLM, "LOCAL memory LLM initialized")
useLocalLLM = true
} catch (e: Exception) {
Log.e(AppConfig.TAG, "Failed to initialize LLM: ${e.message}", e)
Log.e(TAG_LLM, "LOCAL init failed: ${e.message}", e)
@@ -849,35 +990,27 @@ class Live2DChatActivity : AppCompatActivity() {
}
/**
* 使用 LLM 生成回复
* 回忆状态调用本地 LLM，仅用于 memory/what-are-you-thinking
*/
private fun generateResponse(userInput: String) {
private fun requestLocalThought(prompt: String, onResult: (String) -> Unit) {
try {
if (useLocalLLM) {
val systemPrompt = "你是一个友好的数字人助手，回答要简洁明了。"
Log.d(AppConfig.TAG, "Generating response for: $userInput")
val local = llmManager
if (local == null) {
Log.e(TAG_LLM, "LOCAL LLM manager is null, fallback to CLOUD")
cloudApiManager.callLLM(userInput)
return
}
Log.i(TAG_LLM, "LOCAL generateResponseWithSystem")
local.generateResponseWithSystem(systemPrompt, userInput)
} else {
// 使用云端 LLM
Log.d(AppConfig.TAG, "Using cloud LLM for response: $userInput")
Log.i(TAG_LLM, "CLOUD callLLM")
// 调用云端 LLM
cloudApiManager.callLLM(userInput)
val local = llmManager
if (local == null) {
onResult("我在想，下次见面可以聊聊今天的新鲜事。")
return
}
localThoughtSilentMode = true
pendingLocalThoughtCallback = onResult
Log.i(TAG_LLM, "Routing memory thought to LOCAL")
local.generateResponseWithSystem(
"你是数字人内心独白模块，输出一句简短温和的想法。",
prompt
)
} catch (e: Exception) {
Log.e(AppConfig.TAG, "Failed to generate response: ${e.message}", e)
Log.e(TAG_LLM, "generateResponse failed: ${e.message}", e)
runOnUiThread {
uiManager.appendToUi("\n\n[Error] LLM 生成失败: ${e.message}\n")
llmInFlight = false
}
Log.e(TAG_LLM, "requestLocalThought failed: ${e.message}", e)
pendingLocalThoughtCallback = null
localThoughtSilentMode = false
onResult("我在想，下次见面可以聊聊今天的新鲜事。")
}
}
}