diff --git a/app/note/design_doc b/app/note/design_doc index 1fb51e6..023133b 100644 --- a/app/note/design_doc +++ b/app/note/design_doc @@ -270,6 +270,37 @@ https://tianchi.aliyun.com/dataset/93864 解决方法： - 在项目的顶层的 gradle.properties 中添加 tuanjieStreamingAssets 配置 tuanjieStreamingAssets=.unity3d, google-services-desktop.json, google-services.json, GoogleService-Info.plist - - + 4. 问题描述：重新从 Tuanjie 导出 tuanjieLibrary 后报错： + Build was configured to prefer settings repositories over project repositories but repository 'maven' was added by build file 'tuanjieLibrary\build.gradle' + 原因：tuanjieLibrary/build.gradle 里的 repositories {} 块与根目录 settings.gradle 中的 + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) 冲突。 + 解决方法： + - 直接删除 tuanjieLibrary/build.gradle 中的整个 repositories {} 块。 + - 所需的 Aliyun 镜像已在 settings.gradle 的 dependencyResolutionManagement.repositories 中统一声明，无需重复。 + + 5. 问题描述：Build file 'D:\code\digital_person\tuanjieLibrary\build.gradle' line: 6 + + Error resolving plugin [id: 'com.android.application', version: '7.4.2', apply: false] + > The request for this plugin could not be satisfied because the plugin is already on the classpath with a different version (8.2.2). + + 解决方法： + 直接修改项目的 build.gradle 文件，将插件版本改为一致的 8.2.2 + + 6. 问题描述：Build file 'D:\code\digital_person\tuanjieLibrary\build.gradle' line: 16 + + A problem occurred evaluating project ':tuanjieLibrary'. + > Build was configured to prefer settings repositories over project repositories but repository 'maven' was added by build file 'tuanjieLibrary\build.gradle' + + 解决方法（见问题4）：删除 tuanjieLibrary/build.gradle 中的 repositories {} 块。 + + 7. 问题描述：重新从 Tuanjie 导出后，:tuanjieLibrary 模块无法被 :app 解析： + Could not resolve project :tuanjieLibrary. + > No matching configuration of project :tuanjieLibrary was found. None of the consumable configurations have attributes. + 原因：Tuanjie 重新导出后，tuanjieLibrary/build.gradle 被还原为只含 apply false 的"根项目"构建文件， + 没有 apply plugin: 'com.android.library'，Gradle 无法识别该模块为 Android 库。 + 解决方法： + - 将 tuanjieLibrary/build.gradle 替换为完整的 Android library 模块构建文件（apply plugin: 'com.android.library'）， + 参考 launcher/build.gradle 的配置（compileSdkVersion、ndkVersion、ndkPath、aaptOptions、buildTypes 等）， + 并在 dependencies 中添加 implementation files('libs/unity-classes.jar')。 + - 注意：每次从 Tuanjie 重新导出后都需要重新替换该文件。 \ No newline at end of file diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index cdd3b40..bd0f65b 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -5,6 +5,7 @@ + + + + diff --git a/app/src/main/assets/question_prompts.json b/app/src/main/assets/question_prompts.json new file mode 100644 index 0000000..ab63755 --- /dev/null +++ b/app/src/main/assets/question_prompts.json @@ -0,0 +1,67 @@ +[ + { + "subject": "生活数学", + "grade": 1, + "topic": "比大小", + "difficulty": 1, + "promptTemplate": "基于以下3个学习目标，针对一年级小学生出1道题目，题目尽可能是唯一答案，不需要小朋友假设不同的情况来分析给答案，并且提问语言中用到的文字是小学一年级学生学过的，而且题目难度要最简单的，物品是生活中常见的生活用品个体并且是一年级小学生可以经常接触到的，例子：乒乓球和篮球哪个大/小，不要出现大小十分接近和某一部分的物品进行比较，例子：篮球和足球哪个大/小？下面是学习目标：1.初步感知物品的大小，理解大小的含义。2.初步建立大小的概念，会比较2个物品的大小。3.能通过看一看、画一画、拼一拼、比一比等活动比较物品大小" + }, + { + "subject": "生活数学", + "grade": 1, + "topic": "白天、黑夜", + "difficulty": 1, + "promptTemplate": "基于以下2个学习目标，针对一年级小学生出1道题目，题目尽可能是唯一答案，不需要小朋友假设不同的情况来分析给答案，并且提问语言中用到的文字是小学一年级学生学过的，而且题目难度要最简单的：1.能判断白天、黑夜。2.能在情境中区别和分辨白天、黑夜。3.理解白天、黑夜的交替规律。" + }, + { + "subject": "生活数学", + "grade": 1, + "topic": "比多少(1)", + "difficulty": 1, + "promptTemplate": "基于以下2个学习目标，针对一年级小学生出1道题目，题目尽可能是唯一答案，不需要小朋友假设不同的情况来分析给答案，并且提问语言中用到的文字是小学一年级学生学过的，物品是常见的，而且题目难度要最简单的，例子：2 个苹果和 5 个苹果比，谁少？7 颗草莓和 3 颗草莓比，谁多？下面是两个学习目标：1.通过观察生活，感知物品的多少，会区分多少。2.初步建立多少的概念。" + }, + { + "subject": "生活数学", + "grade": 1, + "topic": "比多少(2)", + "difficulty": 1, + "promptTemplate": "基于以下2个学习目标，针对一年级小学生出1道题目，题目尽可能是唯一答案，不需要小朋友假设不同的情况来分析给答案，并且提问语言中用到的文字是小学一年级学生学过的，题目中点物体是生活中常见的，而且题目难度要最简单的，例子：7个衣架和5件衣服，衣架比衣服多，衣服比衣架少，两个物品之间要有关联性，下面是2个学习目标：1.运用一一对应的方法比较物品数量的多少。2.能用对比的方法描述指认物品数量的多少。" + }, + { + "subject": "生活数学", + "grade": 1, + "topic": "认识1", + "difficulty": 1, + "promptTemplate": "基于以下3个学习目标，针对一年级小学生出1道题目，题目尽可能是唯一答案，不需要小朋友假设不同的情况来分析给答案，并且提问语言中用到的文字是小学一年级学生学过的，而且题目难度要最简单的，不要出现这种：数字 0 和数字 1，哪个是数字 1？，正确的是：一个苹果和两个梨，哪个可以用数字1表示。下面是两个学习目标1.能正确认读数字1。2.能手口一致地点数数量是1的物品。3.能正确书写数字1。" + }, + { + "subject": "生活数学", + "grade": 1, + "topic": "认识2", + "difficulty": 1, + "promptTemplate": "基于以下3个学习目标，针对一年级小学生出1道题目，题目尽可能是唯一答案，不需要小朋友假设不同的情况来分析给答案，并且提问语言中用到的文字是小学一年级学生学过的，而且题目难度要最简单，不要出现这种：数字 1 和数字2，哪个是数字 2？，正确的是：一个苹果和两个梨，哪个可以用数字2表示。下面是3个学习目标。1.理解数量是2的物品用数字2表示。2.能数、认、读、写数字2，能手口一致地点数数量是2的物品。3.会在日字格中正确书写数字2。" + }, + { + "subject": "生活数学", + "grade": 1, + "topic": "认识3", + "difficulty": 1, + "promptTemplate": "基于以下4个学习目标，针对一年级小学生出1道题目，题目尽可能是唯一答案，不需要小朋友假设不同的情况来分析给答案，并且提问语言中用到的文字是小学一年级学生学过的，而且题目难度要最简单的：1.在现实情境中，理解数字3的含义，能认读数字3。2.能从数字1点数到数字3。3.能看物点数。4.能按数字3取出相应数量的物品" + }, + { + "subject": "生活数学", + "grade": 1, + "topic": "认识图形", + "difficulty": 1, + "promptTemplate": "基于以下3个学习目标，针对一年级小学生出1道题目，题目尽可能是唯一答案，不需要小朋友假设不同的情况来分析给答案，并且提问语言中用到的文字是小学一年级学生学过的，而且题目难度要最简单的1.初步认识球体，了解球体的特点。2.认识球体的特征。3.能通过看一看、摸一摸、滚一滚等活动，找到生活中的球体。" + }, + { + "subject": "生活数学", + "grade": 1, + "topic": "有、没有", + "difficulty": 1, + "promptTemplate": "基于以下3个学习目标，针对一年级小学生出1道题目，题目尽可能是唯一答案，不需要小朋友假设不同的情况来分析给答案，并且提问语言中用到的文字是小学一年级学生学过的，而且题目难度要最简单的：1.能分辨有和没有。2.能理解有和没有的含义。3.能解决一些生活中有和没有的数学问题。" + } + + +] diff --git a/app/src/main/java/com/digitalperson/DigitalPersonLauncherActivity.kt b/app/src/main/java/com/digitalperson/DigitalPersonLauncherActivity.kt index e4110f7..4da2e2a 100644 --- a/app/src/main/java/com/digitalperson/DigitalPersonLauncherActivity.kt +++ b/app/src/main/java/com/digitalperson/DigitalPersonLauncherActivity.kt @@ -9,14 +9,23 @@ class DigitalPersonLauncherActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) - // 根据配置启动相应的Activity - val intent = if (AppConfig.Avatar.isUnity()) { - Intent(this, UnityDigitalPersonActivity::class.java) - } else { - Intent(this, Live2DChatActivity::class.java) + try { + // 根据配置启动相应的Activity + val intent = if (AppConfig.Avatar.isUnity()) { + android.util.Log.i("DigitalPersonLauncher", "Launching UnityDigitalPersonActivity") + Intent(this, UnityDigitalPersonActivity::class.java) + } else { + android.util.Log.i("DigitalPersonLauncher", "Launching Live2DChatActivity") + Intent(this, Live2DChatActivity::class.java) + } + + startActivity(intent) + finish() + } catch (e: Exception) { + android.util.Log.e("DigitalPersonLauncher", "Failed to launch activity", e) + // 如果启动失败，显示错误信息 + android.widget.Toast.makeText(this, "启动失败: ${e.message}", android.widget.Toast.LENGTH_LONG).show() + finish() } - - startActivity(intent) - finish() } } \ No newline at end of file diff --git a/app/src/main/java/com/digitalperson/Live2DChatActivity.kt b/app/src/main/java/com/digitalperson/Live2DChatActivity.kt index 3d5ed45..6589e10 100644 --- a/app/src/main/java/com/digitalperson/Live2DChatActivity.kt +++ b/app/src/main/java/com/digitalperson/Live2DChatActivity.kt @@ -2,37 +2,30 @@ package com.digitalperson import android.Manifest import android.content.pm.PackageManager -import android.graphics.Bitmap import android.os.Bundle import android.util.Log import android.widget.Toast import androidx.appcompat.app.AlertDialog -import androidx.camera.core.CameraSelector -import com.digitalperson.engine.RetinaFaceEngineRKNN -import com.digitalperson.face.FaceBox -import androidx.camera.core.ImageAnalysis -import androidx.camera.core.ImageProxy -import androidx.camera.core.Preview -import androidx.camera.lifecycle.ProcessCameraProvider import androidx.camera.view.PreviewView import androidx.appcompat.app.AppCompatActivity import androidx.core.app.ActivityCompat import androidx.core.content.ContextCompat import com.digitalperson.cloud.CloudApiManager +import com.digitalperson.cloud.CloudReflectionHelper import com.digitalperson.audio.AudioProcessor import com.digitalperson.vad.VadManager import com.digitalperson.asr.AsrManager import com.digitalperson.ui.Live2DUiManager +import com.digitalperson.question.QuestionGenerationAgent import com.digitalperson.config.AppConfig import com.digitalperson.face.FaceDetectionPipeline import com.digitalperson.face.FaceOverlayView -import com.digitalperson.face.ImageProxyBitmapConverter +import com.digitalperson.face.SharedFaceCameraManager import com.digitalperson.metrics.TraceManager import com.digitalperson.metrics.TraceSession import com.digitalperson.tts.TtsController -import com.digitalperson.interaction.DigitalHumanInteractionController +import com.digitalperson.interaction.BaseDigitalPersonCoordinator import com.digitalperson.data.DatabaseInitializer -import com.digitalperson.interaction.InteractionActionHandler import com.digitalperson.interaction.InteractionState import com.digitalperson.interaction.UserMemoryStore import com.digitalperson.llm.LLMManager @@ -45,9 +38,6 @@ import com.digitalperson.interaction.ConversationSummaryMemory import java.io.File import android.graphics.BitmapFactory -import org.json.JSONObject -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -103,22 +93,19 @@ class Live2DChatActivity : AppCompatActivity() { private lateinit var cameraPreviewView: PreviewView private lateinit var faceOverlayView: FaceOverlayView private lateinit var faceDetectionPipeline: FaceDetectionPipeline - private lateinit var interactionController: DigitalHumanInteractionController + private lateinit var sharedFaceCameraManager: SharedFaceCameraManager + private lateinit var interactionCoordinator: BaseDigitalPersonCoordinator private lateinit var userMemoryStore: UserMemoryStore + private var questionGenerationAgent: QuestionGenerationAgent? = null private lateinit var conversationBufferMemory: ConversationBufferMemory private lateinit var conversationSummaryMemory: ConversationSummaryMemory private var facePipelineReady: Boolean = false - private var cameraProvider: ProcessCameraProvider? = null - private lateinit var cameraAnalyzerExecutor: ExecutorService private var activeUserId: String = "guest" private var pendingLocalThoughtCallback: ((String) -> Unit)? = null private var pendingLocalProfileCallback: ((String) -> Unit)? = null private var localThoughtSilentMode: Boolean = false private val recentConversationLines = ArrayList() private var recentConversationDirty: Boolean = false - private var lastFacePresent: Boolean = false - private var lastFaceIdentityId: String? = null - private var lastFaceRecognizedName: String? = null private lateinit var faceRecognitionTest: FaceRecognitionTest private lateinit var llmSummaryTest: LLMSummaryTest @@ -149,7 +136,7 @@ class Live2DChatActivity : AppCompatActivity() { return } if (facePipelineReady) { - startCameraPreviewAndDetection() + sharedFaceCameraManager.start() } } @@ -176,7 +163,6 @@ class Live2DChatActivity : AppCompatActivity() { cameraPreviewView = findViewById(R.id.camera_preview) cameraPreviewView.implementationMode = PreviewView.ImplementationMode.COMPATIBLE faceOverlayView = findViewById(R.id.face_overlay) - cameraAnalyzerExecutor = Executors.newSingleThreadExecutor() // 初始化数据库 val databaseInitializer = DatabaseInitializer(applicationContext) @@ -185,97 +171,50 @@ class Live2DChatActivity : AppCompatActivity() { userMemoryStore = UserMemoryStore(applicationContext) val database = AppDatabase.getInstance(applicationContext) conversationBufferMemory = ConversationBufferMemory(database) - conversationSummaryMemory = ConversationSummaryMemory(database, llmManager) - interactionController = DigitalHumanInteractionController( - scope = ioScope, - handler = object : InteractionActionHandler { - override fun onStateChanged(state: InteractionState) { - runOnUiThread { - uiManager.appendToUi("\n[State] $state\n") - } - Log.i(TAG_ACTIVITY, "\n[State] $state\n") - if (state == InteractionState.IDLE) { - analyzeUserProfileInIdleIfNeeded() - Log.i(TAG_ACTIVITY, "[analyze] done") - } - } - - override fun playMotion(motionName: String) { - playInteractionMotion(motionName) - } - - override fun appendText(text: String) { - uiManager.appendToUi(text) - } - - override fun speak(text: String) { - ttsController.enqueueSegment(text) - ttsController.enqueueEnd() - } - - override fun requestCloudReply(userText: String) { - llmInFlight = true - Log.i(TAG_LLM, "Routing dialogue to CLOUD") - cloudApiManager.callLLM(buildCloudPromptWithUserProfile(userText)) - } - - override fun requestLocalThought(prompt: String, onResult: (String) -> Unit) { - this@Live2DChatActivity.requestLocalThought(prompt, onResult) - } - - override fun onRememberUser(faceIdentityId: String, name: String?) { - activeUserId = faceIdentityId - userMemoryStore.upsertUserSeen(activeUserId, name) - } - - override fun saveThought(thought: String) { - userMemoryStore.upsertUserSeen(activeUserId, null) - userMemoryStore.updateThought(activeUserId, thought) - } - - override fun loadLatestThought(): String? = userMemoryStore.getLatestThought() - - override fun loadRecentThoughts(timeRangeMs: Long): List = userMemoryStore.getRecentThoughts(timeRangeMs) - - override fun addToChatHistory(role: String, content: String) { - appendConversationLine(role, content) - } - - override fun addAssistantMessageToCloudHistory(content: String) { - cloudApiManager.addAssistantMessage(content) - } - - override fun getRandomQuestion(faceId: String): String { - // 从数据库获取该faceId未被问过的问题 - val question = userMemoryStore.getRandomUnansweredQuestion(faceId) - return question?.content ?: "你喜欢什么颜色呀？" - } - }, - context = applicationContext - ) + conversationSummaryMemory = ConversationSummaryMemory(database, llmManager, CloudReflectionHelper.createCloudGenerator()) + + // 初始化题目生成智能体 + // 如果没有本地LLM，使用云端LLM + if (llmManager != null) { + // 使用本地LLM + val localAgent = QuestionGenerationAgent( + context = applicationContext, + llmManager = llmManager!!, + userMemoryStore = userMemoryStore + ) + questionGenerationAgent = localAgent + Log.i(TAG_ACTIVITY, "QuestionGenerationAgent initialized with local LLM") + } else { + // 使用云端LLM + Log.i(TAG_ACTIVITY, "Initializing QuestionGenerationAgent with cloud LLM") + + val cloudAgent = QuestionGenerationAgent( + context = applicationContext, + llmManager = null, + userMemoryStore = userMemoryStore, + cloudLLMGenerator = CloudReflectionHelper.createCloudGenerator() + ) + questionGenerationAgent = cloudAgent + Log.i(TAG_ACTIVITY, "QuestionGenerationAgent initialized with cloud LLM") + } + questionGenerationAgent?.start() + + interactionCoordinator = createInteractionCoordinator() faceDetectionPipeline = FaceDetectionPipeline( context = applicationContext, onResult = { result -> faceOverlayView.updateResult(result) }, - onPresenceChanged = { present, faceIdentityId, recognizedName -> - if (present != lastFacePresent) { - lastFacePresent = present - Log.d(TAG_ACTIVITY, "presence changed: present=$present") - interactionController.onFacePresenceChanged(present) - if (!present) { - lastFaceIdentityId = null - lastFaceRecognizedName = null - } - } - if (present && (faceIdentityId != lastFaceIdentityId || recognizedName != lastFaceRecognizedName)) { - lastFaceIdentityId = faceIdentityId - lastFaceRecognizedName = recognizedName - Log.d(TAG_ACTIVITY, "identity update: faceIdentityId=$faceIdentityId, recognized=$recognizedName") - interactionController.onFaceIdentityUpdated(faceIdentityId, recognizedName) - } + onPresenceChanged = { present, isFrontal, faceIdentityId, recognizedName -> + interactionCoordinator.onFaceSignal(present, isFrontal, faceIdentityId, recognizedName) } ) + sharedFaceCameraManager = SharedFaceCameraManager( + context = applicationContext, + lifecycleOwner = this, + pipeline = faceDetectionPipeline, + previewView = cameraPreviewView + ) // 根据配置选择交互方式 uiManager.setUseHoldToSpeak(AppConfig.USE_HOLD_TO_SPEAK) @@ -373,20 +312,24 @@ class Live2DChatActivity : AppCompatActivity() { if (success) { Log.i(AppConfig.TAG, "Model files downloaded successfully") uiManager.showToast("模型下载完成", Toast.LENGTH_SHORT) - // 检查本地 LLM 是否可用 + // ⚠️ 本地 LLM 初始化已跳过 + // 但 conversationSummaryMemory 仍然需要初始化（使用云端LLM） + /* if (FileHelper.isLocalLLMAvailable(this)) { Log.i(AppConfig.TAG, "Local LLM is available, enabling local LLM switch") - // 显示本地 LLM 开关，并同步状态 uiManager.showLLMSwitch(false) - // 初始化本地 LLM initLLM() - // 重新初始化 ConversationSummaryMemory conversationSummaryMemory = ConversationSummaryMemory(database, llmManager) - // 启动交互控制器 - interactionController.start() - // 下载完成后初始化其他组件 + interactionCoordinator.start() initializeOtherComponents() } + */ + Log.i(AppConfig.TAG, "Local LLM skipped - using cloud LLM for conversations") + // 初始化 conversationSummaryMemory（llmManager为null，会使用云端） + conversationSummaryMemory = ConversationSummaryMemory(database, llmManager, CloudReflectionHelper.createCloudGenerator()) + // 直接启动交互控制器（使用云端LLM） + interactionCoordinator.start() + initializeOtherComponents() } else { Log.e(AppConfig.TAG, "Failed to download model files: $message") // 显示错误弹窗，阻止应用继续运行 @@ -396,16 +339,21 @@ class Live2DChatActivity : AppCompatActivity() { } ) } else { - // 模型已存在，初始化本地 LLM + // ⚠️ 本地 LLM 初始化已跳过 + // 模型已存在，但跳过本地LLM初始化 + /* initLLM() - // 重新初始化 ConversationSummaryMemory conversationSummaryMemory = ConversationSummaryMemory(database, llmManager) - // 启动交互控制器 - interactionController.start() - // 直接初始化其他组件 + interactionCoordinator.start() initializeOtherComponents() - // 显示本地 LLM 开关，并同步状态 uiManager.showLLMSwitch(false) + */ + Log.i(AppConfig.TAG, "Local LLM skipped - using cloud LLM for conversations") + // 初始化 conversationSummaryMemory（llmManager为null，会使用云端） + conversationSummaryMemory = ConversationSummaryMemory(database, llmManager, CloudReflectionHelper.createCloudGenerator()) + // 直接启动交互控制器（使用云端LLM） + interactionCoordinator.start() + initializeOtherComponents() } @@ -434,7 +382,7 @@ class Live2DChatActivity : AppCompatActivity() { if (!facePipelineReady) { uiManager.showToast("RetinaFace 初始化失败，请检查模型和 rknn 运行库", Toast.LENGTH_LONG) } else if (allPermissionsGranted()) { - startCameraPreviewAndDetection() + sharedFaceCameraManager.start() } uiManager.setText(getString(R.string.hint)) if (AppConfig.USE_HOLD_TO_SPEAK) { @@ -516,6 +464,91 @@ class Live2DChatActivity : AppCompatActivity() { } } + private fun createInteractionCoordinator(): BaseDigitalPersonCoordinator { + return object : BaseDigitalPersonCoordinator( + scope = ioScope, + context = applicationContext, + userMemoryStore = userMemoryStore, + conversationBufferMemory = conversationBufferMemory, + conversationSummaryMemory = conversationSummaryMemory, + llmManager = llmManager, + cloudLLMGenerator = CloudReflectionHelper.createCloudGenerator(), + ) { + override fun onStateChangedInternal(state: InteractionState) { + super.onStateChangedInternal(state) // triggers analyzeUserProfileInIdleIfNeeded + runOnUiThread { + uiManager.appendToUi("\n[State] $state\n") + } + Log.i(TAG_ACTIVITY, "\n[State] $state\n") + playMotionForState(state) + playMoodForState(state) + } + + override fun onProfileUpdated(userId: String) { + runOnUiThread { + uiManager.appendToUi("\n[Memory] 已更新用户画像: $userId\n") + } + } + + override fun onActiveUserChanged(faceId: String, recognizedName: String?) { + activeUserId = faceId + } + + override fun onLlmInFlightChanged(inFlight: Boolean) { + llmInFlight = inFlight + } + + override fun onAppendText(text: String) { + uiManager.appendToUi(text) + } + + override fun onSpeak(text: String) { + ttsController.enqueueSegment(text) + ttsController.enqueueEnd() + } + + override fun onRequestCloudReply(prompt: String) { + Log.i(TAG_LLM, "Routing dialogue to CLOUD") + cloudApiManager.callLLM(prompt) + } + + override fun onRequestLocalThought(prompt: String, onResult: (String) -> Unit) { + this@Live2DChatActivity.requestLocalThought(prompt, onResult) + } + + override fun onAddConversation(role: String, content: String) { + val uiRole = if (role.equals("assistant", true) || role == "助手") "助手" else "用户" + appendConversationLine(uiRole, content) + } + + override fun onAddAssistantMessageToCloudHistory(content: String) { + cloudApiManager.addAssistantMessage(content) + } + + override fun onGetRandomQuestion(faceId: String): String { + val question = userMemoryStore.getRandomUnansweredQuestion(faceId) + return question?.content ?: "你喜欢什么颜色呀？" + } + + override fun onQuestionAskedCallback(userId: String) { + // 每次提问后触发题目生成检查 + questionGenerationAgent?.onQuestionAsked(userId) + ?: Log.w(TAG_ACTIVITY, "QuestionGenerationAgent not initialized, skipping replenishment") + } + + override fun onFaceAppearedCallback(userId: String) { + // 人脸出现时立即触发题目预生成检查 + Log.i(TAG_ACTIVITY, "Face appeared, triggering question pre-generation for user: $userId") + questionGenerationAgent?.onQuestionAsked(userId) + ?: Log.w(TAG_ACTIVITY, "QuestionGenerationAgent not initialized yet, skipping question generation") + } + + override fun buildCloudPromptWithUserProfile(userText: String): String { + return this@Live2DChatActivity.buildCloudPromptWithUserProfile(userText) + } + } + } + private fun createAsrCallback() = object : AsrManager.AsrCallback { override fun onAsrStarted() { @@ -545,7 +578,7 @@ class Live2DChatActivity : AppCompatActivity() { override fun onLlmCalled(text: String) { Log.d(AppConfig.TAG, "Forward ASR text to interaction controller: $text") - interactionController.onUserAsrText(text) + interactionCoordinator.onUserAsrText(text) } } @@ -564,7 +597,6 @@ class Live2DChatActivity : AppCompatActivity() { override fun onLLMResponseReceived(response: String) { currentTrace?.markLlmDone() llmInFlight = false - appendConversationLine("助手", response) if (enableStreaming) { for (seg in segmenter.flush()) { @@ -572,21 +604,15 @@ class Live2DChatActivity : AppCompatActivity() { } ttsController.enqueueEnd() } else { - val previousMood = com.digitalperson.mood.MoodManager.getCurrentMood() - val (filteredText, mood) = com.digitalperson.mood.MoodManager.extractAndFilterMood(response) - android.util.Log.d(com.digitalperson.config.AppConfig.TAG, "Final mood: $mood, filtered text: $filteredText") - - if (mood != previousMood) { + val filteredText = ttsController.speakLlmResponse(response) { mood -> + android.util.Log.d(com.digitalperson.config.AppConfig.TAG, "Mood changed to: $mood") uiManager.setMood(mood) } - runOnUiThread { - uiManager.appendToUi("${filteredText}\n") + uiManager.appendToUi("${filteredText.orEmpty()}\n") } - ttsController.enqueueSegment(filteredText) - ttsController.enqueueEnd() } - interactionController.onDialogueResponseFinished() + interactionCoordinator.onCloudFinalResponse(response) } override fun onLLMStreamingChunkReceived(chunk: String) { @@ -618,7 +644,7 @@ class Live2DChatActivity : AppCompatActivity() { override fun onError(errorMessage: String) { llmInFlight = false uiManager.showToast(errorMessage, Toast.LENGTH_LONG) - interactionController.onDialogueResponseFinished() + interactionCoordinator.onCloudError() onStopClicked(userInitiated = false) } } @@ -634,6 +660,7 @@ class Live2DChatActivity : AppCompatActivity() { runOnUiThread { uiManager.appendToUi("\n[LOG] TTS completed at: ${System.currentTimeMillis()}\n") } + interactionCoordinator.onTtsPlaybackCompleted() } override fun onTtsSegmentCompleted(durationMs: Long) {} @@ -658,8 +685,8 @@ class Live2DChatActivity : AppCompatActivity() { override fun onDestroy() { super.onDestroy() - try { interactionController.stop() } catch (_: Throwable) {} - stopCameraPreviewAndDetection() + try { interactionCoordinator.stop() } catch (_: Throwable) {} + sharedFaceCameraManager.stop() onStopClicked(userInitiated = false) ioScope.cancel() synchronized(nativeLock) { @@ -667,7 +694,7 @@ class Live2DChatActivity : AppCompatActivity() { try { asrManager.release() } catch (_: Throwable) {} } try { faceDetectionPipeline.release() } catch (_: Throwable) {} - try { cameraAnalyzerExecutor.shutdown() } catch (_: Throwable) {} + try { sharedFaceCameraManager.release() } catch (_: Throwable) {} try { ttsController.release() } catch (_: Throwable) {} try { llmManager?.destroy() } catch (_: Throwable) {} try { uiManager.release() } catch (_: Throwable) {} @@ -679,13 +706,13 @@ class Live2DChatActivity : AppCompatActivity() { Log.i(TAG_ACTIVITY, "onResume") uiManager.onResume() if (facePipelineReady && allPermissionsGranted()) { - startCameraPreviewAndDetection() + sharedFaceCameraManager.start() } } override fun onPause() { Log.i(TAG_ACTIVITY, "onPause") - stopCameraPreviewAndDetection() + sharedFaceCameraManager.stop() uiManager.onPause() super.onPause() } @@ -696,60 +723,6 @@ class Live2DChatActivity : AppCompatActivity() { } } - private fun startCameraPreviewAndDetection() { - val cameraProviderFuture = ProcessCameraProvider.getInstance(this) - cameraProviderFuture.addListener({ - try { - val provider = cameraProviderFuture.get() - cameraProvider = provider - provider.unbindAll() - - val preview = Preview.Builder().build().apply { - setSurfaceProvider(cameraPreviewView.surfaceProvider) - } - cameraPreviewView.scaleType = PreviewView.ScaleType.FIT_CENTER - - val analyzer = ImageAnalysis.Builder() - .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) - .build() - - analyzer.setAnalyzer(cameraAnalyzerExecutor) { imageProxy -> - analyzeCameraFrame(imageProxy) - } - - val selector = CameraSelector.Builder() - .requireLensFacing(CameraSelector.LENS_FACING_FRONT) - .build() - - provider.bindToLifecycle(this, selector, preview, analyzer) - } catch (t: Throwable) { - Log.e(AppConfig.TAG, "startCameraPreviewAndDetection failed: ${t.message}", t) - } - }, ContextCompat.getMainExecutor(this)) - } - - private fun stopCameraPreviewAndDetection() { - try { - cameraProvider?.unbindAll() - } catch (_: Throwable) { - } finally { - cameraProvider = null - } - } - - private fun analyzeCameraFrame(imageProxy: ImageProxy) { - try { - val bitmap: Bitmap? = ImageProxyBitmapConverter.toBitmap(imageProxy) - if (bitmap != null) { - faceDetectionPipeline.submitFrame(bitmap) - } - } catch (t: Throwable) { - Log.w(AppConfig.TAG, "analyzeCameraFrame error: ${t.message}") - } finally { - imageProxy.close() - } - } - private fun onStartClicked() { Log.d(AppConfig.TAG, "onStartClicked called") if (isRecording) { @@ -799,7 +772,7 @@ class Live2DChatActivity : AppCompatActivity() { } // 通知状态机用户开始说话，立即进入对话状态 - interactionController.onUserStartSpeaking() + interactionCoordinator.onUserStartSpeaking() if (!audioProcessor.initMicrophone(micPermissions, AppConfig.REQUEST_RECORD_AUDIO_PERMISSION)) { uiManager.showToast("麦克风初始化失败/无权限") @@ -941,16 +914,6 @@ class Live2DChatActivity : AppCompatActivity() { Log.d(AppConfig.TAG, "processSamplesLoop stopped") } - private fun playInteractionMotion(motionName: String) { - when (motionName) { - "haru_g_m22.motion3.json" -> uiManager.setMood("高兴") - "haru_g_m01.motion3.json", "haru_g_m17.motion3.json" -> uiManager.setMood("中性") - "haru_g_m15.motion3.json" -> uiManager.setMood("关心") - "haru_g_idle.motion3.json" -> uiManager.setMood("平和") - else -> uiManager.setMood("中性") - } - } - private fun appendConversationLine(role: String, text: String) { val line = "$role: ${text.trim()}" if (line.length <= 4) return @@ -993,136 +956,7 @@ class Live2DChatActivity : AppCompatActivity() { } } - private fun analyzeUserProfileInIdleIfNeeded() { - if (!activeUserId.startsWith("face_")) { - Log.d(AppConfig.TAG, "faceID is not face_") - return - } - - // 使用 conversationBufferMemory 获取对话消息 - val messages = conversationBufferMemory.getMessages(activeUserId) - Log.d(AppConfig.TAG, "msg is empty? ${messages.isEmpty()}") - val hasUserMessages = messages.any { it.role == "user" } - Log.d(AppConfig.TAG, "msg has user messages? $hasUserMessages") - if (messages.isEmpty() || !hasUserMessages) return - - // 生成对话摘要 - conversationSummaryMemory.generateSummary(activeUserId, messages) { summary -> - Log.d(AppConfig.TAG, "Generated conversation summary for $activeUserId: $summary") - } - - // 使用多角度提问方式提取用户信息 - val dialogue = messages.joinToString("\n") { "${it.role}: ${it.content}" } - - requestMultiAngleProfileExtraction(dialogue) { profileData -> - try { - val nameToUpdate = profileData["name"]?.trim()?.ifBlank { null } - val ageToUpdate = profileData["age"]?.trim()?.ifBlank { null } - val genderToUpdate = profileData["gender"]?.trim()?.ifBlank { null } - val hobbiesToUpdate = profileData["hobbies"]?.trim()?.ifBlank { null } - val summaryToUpdate = profileData["summary"]?.trim()?.ifBlank { null } - Log.d(TAG_LLM, "profileData: $profileData") - if (nameToUpdate != null || ageToUpdate != null || genderToUpdate != null || hobbiesToUpdate != null || summaryToUpdate != null) { - if (nameToUpdate != null) { - userMemoryStore.updateDisplayName(activeUserId, nameToUpdate) - Log.i(TAG_LLM, "Updated display name to $nameToUpdate") - } - userMemoryStore.updateProfile(activeUserId, ageToUpdate, genderToUpdate, hobbiesToUpdate, summaryToUpdate) - - // 清空已处理的对话记录 - conversationBufferMemory.clear(activeUserId) - - runOnUiThread { - uiManager.appendToUi("\n[Memory] 已更新用户画像: $activeUserId\n") - } - } - } catch (e: Exception) { - Log.w(TAG_LLM, "Profile parse failed: ${e.message}") - } - } - } - - private fun requestMultiAngleProfileExtraction(dialogue: String, onResult: (Map) -> Unit) { - try { - val local = llmManager - if (local == null) { - onResult(emptyMap()) - return - } - - val questions = listOf( - "请从对话中提取用户的姓名，只返回姓名，如果没有提到姓名，请返回未知", - "请从对话中提取用户的年龄，只返回年龄，如果没有提到年龄，请返回未知", - "请从对话中提取用户的性别，只返回性别，如果没有提到性别，请返回未知", - "请从对话中提取用户的爱好，只返回爱好，如果没有提到爱好，请返回未知", - "请总结对话，只返回总结的内容" - ) - - var completed = 0 - val results = mutableMapOf() - - questions.forEach { question -> - val prompt = buildMultiAnglePrompt(dialogue, question) - local.generate(prompt) { answer -> - val processedAnswer = processProfileAnswer(answer) - - when { - question.contains("姓名") -> results["name"] = processedAnswer - question.contains("年龄") -> results["age"] = processedAnswer - question.contains("性别") -> results["gender"] = processedAnswer - question.contains("爱好") -> results["hobbies"] = processedAnswer - question.contains("总结") -> results["summary"] = processedAnswer - } - - completed++ - - if (completed == questions.size) { - onResult(results) - } - } - } - } catch (e: Exception) { - Log.e(TAG_LLM, "requestMultiAngleProfileExtraction failed: ${e.message}", e) - onResult(emptyMap()) - } - } - private fun buildMultiAnglePrompt(dialogue: String, question: String): String { - return """ - 请根据以下对话回答问题： - - 对话内容： - $dialogue - - 问题：$question - - 回答： - """.trimIndent() - } - - private fun processProfileAnswer(answer: String): String { - var processed = answer.replace("<", "").replace(">", "") - if (processed.contains("unknown", ignoreCase = true) || - processed.contains("null", ignoreCase = true) || - processed.contains("未知")) { - return "" - } - if (processed.contains(":")) { - processed = processed.substringAfter(":").trim() - } - processed = processed.replace(".", "").trim() - return processed - } - - private fun parseFirstJsonObject(text: String): JSONObject { - val raw = text.trim() - val start = raw.indexOf('{') - val end = raw.lastIndexOf('}') - if (start >= 0 && end > start) { - return JSONObject(raw.substring(start, end + 1)) - } - return JSONObject(raw) - } /** * 初始化本地 LLM（仅用于回忆状态） @@ -1170,9 +1004,10 @@ class Live2DChatActivity : AppCompatActivity() { return@runOnUiThread } if (!localThoughtSilentMode && finalText.isNotEmpty()) { - uiManager.appendToUi("$finalText\n") - ttsController.enqueueSegment(finalText) - ttsController.enqueueEnd() + val filteredText = ttsController.speakLlmResponse(finalText) { mood -> + uiManager.setMood(mood) + } + uiManager.appendToUi("${filteredText.orEmpty()}\n") } localThoughtSilentMode = false } @@ -1232,4 +1067,33 @@ class Live2DChatActivity : AppCompatActivity() { onResult("我在想，下次见面可以聊聊今天的新鲜事。") } } + + private fun playMotionForState(state: InteractionState) { + // Map states to specific motion names that exist in Live2D assets + val motionName = when (state) { + InteractionState.GREETING -> "haru_g_m22" // waving motion + InteractionState.WAITING_REPLY -> "haru_g_m17" // listening motion + InteractionState.PROACTIVE -> "haru_g_m15" // acknowledging motion + InteractionState.FAREWELL -> "haru_g_m01" // goodbye waving + else -> null // IDLE, MEMORY, DIALOGUE don't trigger specific motions + } + if (motionName != null) { + uiManager.startSpecificMotion(motionName) + } + } + + private fun playMoodForState(state: InteractionState) { + // Mood affects facial expressions only + val mood = when (state) { + InteractionState.IDLE -> "平和" + InteractionState.MEMORY -> "平和" + InteractionState.GREETING -> "高兴" + InteractionState.WAITING_REPLY -> "中性" + InteractionState.DIALOGUE -> "中性" + InteractionState.PROACTIVE -> "关心" + InteractionState.FAREWELL -> "平和" + } + uiManager.setMood(mood) + } + } \ No newline at end of file diff --git a/app/src/main/java/com/digitalperson/MainActivity.kt b/app/src/main/java/com/digitalperson/MainActivity.kt index 7bbe5c0..a3360f6 100644 --- a/app/src/main/java/com/digitalperson/MainActivity.kt +++ b/app/src/main/java/com/digitalperson/MainActivity.kt @@ -210,14 +210,11 @@ class MainActivity : AppCompatActivity() { } ttsManager.enqueueEnd() } else { - val (filteredText, mood) = com.digitalperson.mood.MoodManager.extractAndFilterMood(response) - android.util.Log.d(com.digitalperson.config.AppConfig.TAG, "Final mood: $mood, filtered text: $filteredText") - + val filteredText = ttsManager.speakLlmResponse(response) + android.util.Log.d(com.digitalperson.config.AppConfig.TAG, "LLM response filtered: ${filteredText?.take(60)}") runOnUiThread { - uiManager.appendToUi("${filteredText}\n") + uiManager.appendToUi("${filteredText.orEmpty()}\n") } - ttsManager.enqueueSegment(filteredText) - ttsManager.enqueueEnd() } } diff --git a/app/src/main/java/com/digitalperson/QuestionGenerationTestActivity.kt b/app/src/main/java/com/digitalperson/QuestionGenerationTestActivity.kt new file mode 100644 index 0000000..64842a4 --- /dev/null +++ b/app/src/main/java/com/digitalperson/QuestionGenerationTestActivity.kt @@ -0,0 +1,327 @@ +package com.digitalperson + +import android.os.Bundle +import android.util.Log +import android.widget.Button +import android.widget.LinearLayout +import android.widget.ScrollView +import android.widget.TextView +import android.widget.Toast +import androidx.appcompat.app.AppCompatActivity +import com.digitalperson.config.AppConfig +import com.digitalperson.data.AppDatabase +import com.digitalperson.interaction.UserMemoryStore +import com.digitalperson.llm.LLMManager +import com.digitalperson.llm.LLMManagerCallback +import com.digitalperson.question.QuestionGenerationAgent +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch + +/** + * 题目生成测试Activity + * 用于快速测试LLM题目生成功能，不需要完整启动主应用 + * + * 使用方法： + * 1. 在AndroidManifest.xml中注册这个Activity + * 2. 启动应用时直接打开这个Activity进行测试 + */ +class QuestionGenerationTestActivity : AppCompatActivity() { + + private lateinit var outputText: TextView + private lateinit var llmManager: LLMManager + private lateinit var userMemoryStore: UserMemoryStore + private lateinit var questionGenerationAgent: QuestionGenerationAgent + private val testScope = CoroutineScope(Dispatchers.Main) + + companion object { + private const val TAG = "QuestionGenTest" + } + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setupUI() + initializeComponents() + } + + private fun setupUI() { + val scrollView = ScrollView(this).apply { + layoutParams = LinearLayout.LayoutParams( + LinearLayout.LayoutParams.MATCH_PARENT, + LinearLayout.LayoutParams.MATCH_PARENT + ) + setPadding(32, 32, 32, 32) + } + + val layout = LinearLayout(this).apply { + orientation = LinearLayout.VERTICAL + layoutParams = LinearLayout.LayoutParams( + LinearLayout.LayoutParams.MATCH_PARENT, + LinearLayout.LayoutParams.WRAP_CONTENT + ) + } + + // 标题 + TextView(this).apply { + text = "题目生成测试工具" + textSize = 24f + setPadding(0, 0, 0, 32) + layout.addView(this) + } + + // 输出区域 + outputText = TextView(this).apply { + textSize = 14f + setPadding(16, 16, 16, 16) + background = getDrawable(android.R.drawable.edit_text) + minLines = 20 + layout.addView(this) + } + + // 按钮1: 测试加载提示词池 + Button(this).apply { + text = "1. 加载提示词池" + setPadding(0, 32, 0, 16) + setOnClickListener { testLoadPrompts() } + layout.addView(this) + } + + // 按钮2: 测试生成单个题目 + Button(this).apply { + text = "2. 生成单个题目 (测试LLM)" + setPadding(0, 16, 0, 16) + setOnClickListener { testGenerateOneQuestion() } + layout.addView(this) + } + + // 按钮3: 触发批量生成 + Button(this).apply { + text = "3. 批量生成题目 (启动Agent)" + setPadding(0, 16, 0, 16) + setOnClickListener { testBatchGeneration() } + layout.addView(this) + } + + // 按钮4: 查看数据库题目 + Button(this).apply { + text = "4. 查看数据库题目" + setPadding(0, 16, 0, 16) + setOnClickListener { viewDatabaseQuestions() } + layout.addView(this) + } + + // 按钮5: 清空数据库 + Button(this).apply { + text = "5. 清空测试题目" + setPadding(0, 16, 0, 16) + setOnClickListener { clearTestQuestions() } + layout.addView(this) + } + + scrollView.addView(layout) + setContentView(scrollView) + } + + private fun initializeComponents() { + appendOutput("🔧 初始化组件...") + + try { + // 初始化LLM + val llmCallback = object : LLMManagerCallback { + override fun onThinking(msg: String, finished: Boolean) { + // Test activity doesn't need thinking updates + } + + override fun onResult(msg: String, finished: Boolean) { + // Test activity handles results via generate() callback + } + } + llmManager = LLMManager(AppConfig.LLM.MODEL_DIR, llmCallback) + appendOutput("✅ LLM初始化成功 (模型: ${AppConfig.LLM.MODEL_DIR})") + + // 初始化用户存储 + userMemoryStore = UserMemoryStore(this) + appendOutput("✅ UserMemoryStore初始化成功") + + // 初始化题目生成智能体 + questionGenerationAgent = QuestionGenerationAgent( + context = this, + llmManager = llmManager, + userMemoryStore = userMemoryStore + ) + questionGenerationAgent.start() + appendOutput("✅ QuestionGenerationAgent启动成功") + + appendOutput("\n🎉 所有组件初始化完成！可以开始测试了。\n") + } catch (e: Exception) { + appendOutput("❌ 初始化失败: ${e.message}") + Log.e(TAG, "Initialization failed", e) + } + } + + private fun testLoadPrompts() { + appendOutput("\n" + "=".repeat(50)) + appendOutput("测试1: 加载提示词池") + appendOutput("=".repeat(50)) + + try { + val inputStream = assets.open("question_prompts.json") + val jsonString = inputStream.bufferedReader().use { it.readText() } + val jsonArray = org.json.JSONArray(jsonString) + + appendOutput("✅ 成功加载 ${jsonArray.length()} 个提示词\n") + + for (i in 0 until jsonArray.length()) { + val json = jsonArray.getJSONObject(i) + val subject = json.getString("subject") + val topic = json.getString("topic") + val difficulty = json.getInt("difficulty") + appendOutput(" [$i] $subject / $topic (难度:$difficulty)") + } + + appendOutput("\n✅ 测试完成\n") + } catch (e: Exception) { + appendOutput("❌ 加载失败: ${e.message}") + Log.e(TAG, "Failed to load prompts", e) + } + } + + private fun testGenerateOneQuestion() { + appendOutput("\n" + "=".repeat(50)) + appendOutput("测试2: 生成单个题目 (调用LLM)") + appendOutput("=".repeat(50)) + appendOutput("⏳ 正在生成题目，请稍候...\n") + + val testPrompt = """ + 你是一个专门为特殊教育儿童设计题目的教育专家。请根据以下要求生成一个题目： + + 学科：生活数学 + 年级：1 + 主题：比大小 + 难度：1 + + 具体要求： + 基于以下学习目标，针对一年级小学生出1道题目，题目尽可能是唯一答案， + 并且提问语言中用到的文字是小学一年级学生学过的，而且题目难度要最简单的。 + 例子：乒乓球和篮球哪个大/小 + + 请以JSON格式返回，格式如下： + { + "content": "题目内容", + "answer": "标准答案" + } + + 只返回JSON，不要其他内容。 + """.trimIndent() + + llmManager.generate(testPrompt) { response -> + runOnUiThread { + appendOutput("📝 LLM响应:\n$response\n") + + try { + val json = extractJsonFromResponse(response) + if (json != null) { + val content = json.getString("content") + val answer = json.getString("answer") + appendOutput("✅ 解析成功!") + appendOutput("题目: $content") + appendOutput("答案: $answer\n") + } else { + appendOutput("❌ 无法解析JSON响应\n") + } + } catch (e: Exception) { + appendOutput("❌ 解析失败: ${e.message}\n") + } + } + } + } + + private fun testBatchGeneration() { + appendOutput("\n" + "=".repeat(50)) + appendOutput("测试3: 批量生成题目") + appendOutput("=".repeat(50)) + + val testUserId = "test_user_001" + appendOutput("⏳ 触发题目生成 for user: $testUserId\n") + + testScope.launch { + questionGenerationAgent.triggerGeneration(testUserId) + appendOutput("✅ 生成任务已启动，请查看日志输出\n") + } + } + + private fun viewDatabaseQuestions() { + appendOutput("\n" + "=".repeat(50)) + appendOutput("测试4: 查看数据库题目") + appendOutput("=".repeat(50)) + + val database = AppDatabase.getInstance(this) + val questionDao = database.questionDao() + + // 获取所有题目 + val allQuestions = questionDao.getQuestionsByGrade(1) + + appendOutput("📊 数据库中共有 ${allQuestions.size} 个一年级题目\n") + + allQuestions.take(10).forEachIndexed { index, question -> + appendOutput("[$index] ${question.subject} - ${question.content.take(30)}...") + appendOutput(" 答案: ${question.answer ?: "无"}") + appendOutput(" 难度: ${question.difficulty}\n") + } + + if (allQuestions.size > 10) { + appendOutput("... 还有 ${allQuestions.size - 10} 个题目\n") + } + + // 统计 + val subjectCount = allQuestions.groupBy { it.subject } + appendOutput("\n📈 题目分布:") + subjectCount.forEach { (subject, questions) -> + appendOutput(" $subject: ${questions.size} 个") + } + + appendOutput("\n✅ 查看完成\n") + } + + private fun clearTestQuestions() { + appendOutput("\n" + "=".repeat(50)) + appendOutput("测试5: 清空测试题目") + appendOutput("=".repeat(50)) + + val database = AppDatabase.getInstance(this) + val questionDao = database.questionDao() + + // 这里只是示例，实际清空需要谨慎 + appendOutput("⚠️ 危险操作！确定要清空所有题目吗？") + appendOutput("请在代码中确认后再执行\n") + + Toast.makeText(this, "清空功能需要在代码中确认", Toast.LENGTH_LONG).show() + } + + private fun appendOutput(text: String) { + runOnUiThread { + outputText.append("$text\n") + // 滚动到底部 + outputText.post { + val scrollAmount = outputText.layout?.getLineTop(outputText.lineCount) ?: 0 + outputText.parent?.let { parent -> + if (parent is ScrollView) { + parent.scrollTo(0, scrollAmount) + } + } + } + } + } + + private fun extractJsonFromResponse(response: String): org.json.JSONObject? { + val trimmed = response.trim() + val start = trimmed.indexOf('{') + val end = trimmed.lastIndexOf('}') + + if (start >= 0 && end > start) { + val jsonStr = trimmed.substring(start, end + 1) + return org.json.JSONObject(jsonStr) + } + return null + } +} diff --git a/app/src/main/java/com/digitalperson/UnityDigitalPersonActivity.kt b/app/src/main/java/com/digitalperson/UnityDigitalPersonActivity.kt index 26f1e4b..ee46bad 100644 --- a/app/src/main/java/com/digitalperson/UnityDigitalPersonActivity.kt +++ b/app/src/main/java/com/digitalperson/UnityDigitalPersonActivity.kt @@ -1,29 +1,45 @@ package com.digitalperson import android.Manifest +import android.animation.ObjectAnimator +import android.content.Context import android.content.pm.PackageManager +import android.content.res.ColorStateList +import android.graphics.Color +import android.os.Build import android.os.Bundle import android.os.Handler import android.os.Looper +import android.os.VibrationEffect +import android.os.Vibrator import android.util.Log import android.view.MotionEvent import android.view.ViewGroup import android.widget.Button -import android.widget.EditText import android.widget.TextView +import android.widget.Toast import androidx.core.app.ActivityCompat import androidx.core.content.ContextCompat import android.util.Base64 import android.view.View +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleOwner +import androidx.lifecycle.LifecycleRegistry import com.unity3d.player.UnityPlayer import com.unity3d.player.UnityPlayerActivity import com.digitalperson.audio.AudioProcessor import com.digitalperson.asr.AsrManager +import com.digitalperson.cloud.CloudApiManager +import com.digitalperson.cloud.CloudReflectionHelper import com.digitalperson.config.AppConfig +import com.digitalperson.question.QuestionGenerationAgent import com.digitalperson.data.AppDatabase import com.digitalperson.face.FaceDetectionPipeline +import com.digitalperson.face.SharedFaceCameraManager +import com.digitalperson.interaction.BaseDigitalPersonCoordinator import com.digitalperson.interaction.ConversationBufferMemory import com.digitalperson.interaction.ConversationSummaryMemory +import com.digitalperson.interaction.InteractionState import com.digitalperson.interaction.UserMemoryStore import com.digitalperson.llm.LLMManager import com.digitalperson.llm.LLMManagerCallback @@ -32,29 +48,66 @@ import com.digitalperson.util.FileHelper import com.digitalperson.vad.VadManager import kotlinx.coroutines.* -class UnityDigitalPersonActivity : UnityPlayerActivity() { +class UnityDigitalPersonActivity : UnityPlayerActivity(), LifecycleOwner { // ==================== 伴生对象（静态成员）==================== companion object { - private var instance: UnityDigitalPersonActivity? = null - + private const val TAG = "UnityDigitalPerson" + @Volatile var instance: UnityDigitalPersonActivity? = null + } + // ==================== Unity实例方法（JNI实例调用）==================== + // Unity C# calls these via AndroidJavaObject.Call() on the activity instance. + // @JvmStatic companion methods are NOT visible to AndroidJavaObject.Call() — only + // true instance methods on the class are found by JNI instance dispatch. + /** + * Called by Unity C# via: + * activity.Call("setUnityAudioTarget", gameObjectName) + * Sets the Unity GameObject name that receives TTS audio and motion events. + */ + fun setUnityAudioTarget(gameObjectName: String) { + setUnityAudioTargetInternal(gameObjectName) + } + /** + * Called by Unity C# via: + * activity.Call("setTTSCallback", runnable) + * Registers a Runnable invoked when TTS playback starts. + * Passing null unregisters the callback. + */ + fun setTTSCallback(callback: Runnable?) { + ttsCallback = callback + Log.d(TAG, "TTS callback registered: ${callback != null}") + } + + /** + * Called by Unity C# via: + * activity.Call("setTTSStopCallback", runnable) + * Registers a Runnable invoked when TTS playback stops. + * Passing null unregisters the callback. + */ + fun setTTSStopCallback(callback: Runnable?) { + ttsStopCallback = callback + Log.d(TAG, "TTS stop callback registered: ${callback != null}") } // ==================== 核心模块 ==================== private lateinit var conversationBufferMemory: ConversationBufferMemory private lateinit var conversationSummaryMemory: ConversationSummaryMemory private var llmManager: LLMManager? = null + private lateinit var cloudApiManager: CloudApiManager + private lateinit var interactionCoordinator: BaseDigitalPersonCoordinator private lateinit var faceDetectionPipeline: FaceDetectionPipeline + private lateinit var sharedFaceCameraManager: SharedFaceCameraManager private lateinit var userMemoryStore: UserMemoryStore + private var questionGenerationAgent: QuestionGenerationAgent? = null + private var facePipelineReady: Boolean = false private lateinit var chatHistoryText: TextView private lateinit var holdToSpeakButton: Button - private lateinit var messageInput: EditText - private lateinit var sendButton: Button - + private var recordButtonGlow: View? = null + private var pulseAnimator: ObjectAnimator? = null // 音频和AI模块 @@ -66,11 +119,9 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { // ==================== 状态标志 ==================== @Volatile private var isRecording: Boolean = false - - @Volatile - private var llmInFlight: Boolean = false - - private var useLocalLLM = false // 默认使用云端 LLM + + private var pendingLocalThoughtCallback: ((String) -> Unit)? = null + private var localThoughtSilentMode: Boolean = false // ==================== TTS回调相关 ==================== private var isTTSPlaying = false @@ -81,21 +132,10 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { private var ttsStopCallback: Runnable? = null private var unityAudioTargetObject: String = "DigitalPerson" - // 非静态方法，供Unity调用 - fun setUnityAudioTarget(gameObjectName: String) { + // 非静态方法，内部使用 + private fun setUnityAudioTargetInternal(gameObjectName: String) { unityAudioTargetObject = gameObjectName - Log.d("UnityDigitalPerson", "Unity audio target set: $gameObjectName") - } - - fun setTTSCallback(callback: Runnable) { - instance?.ttsCallback = callback - Log.d("UnityDigitalPerson", "TTS callback registered") - } - - - fun setTTSStopCallback(callback: Runnable) { - instance?.ttsStopCallback = callback - Log.d("UnityDigitalPerson", "TTS stop callback registered") + Log.d(TAG, "Unity audio target set: $gameObjectName") } // ==================== 音频处理 ==================== @@ -108,12 +148,19 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { private var asrWorkerJob: Job? = null // ==================== 权限 ==================== + private val appPermissions: Array = arrayOf( + Manifest.permission.RECORD_AUDIO, + Manifest.permission.CAMERA + ) private val micPermissions: Array = arrayOf(Manifest.permission.RECORD_AUDIO) - private val cameraPermissions: Array = arrayOf(Manifest.permission.CAMERA) + private val lifecycleRegistry = LifecycleRegistry(this) + + override fun getLifecycle(): Lifecycle = lifecycleRegistry // ==================== 生命周期 ==================== override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) + lifecycleRegistry.currentState = Lifecycle.State.CREATED // 设置单例实例 instance = this @@ -126,16 +173,44 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { // 初始化所有组件 initComponents() } + + override fun onStart() { + super.onStart() + lifecycleRegistry.currentState = Lifecycle.State.STARTED + } + + override fun onResume() { + super.onResume() + lifecycleRegistry.currentState = Lifecycle.State.RESUMED + if (facePipelineReady && allPermissionsGranted()) { + sharedFaceCameraManager.start() + } + } + + override fun onPause() { + sharedFaceCameraManager.stop() + lifecycleRegistry.currentState = Lifecycle.State.STARTED + super.onPause() + } + + override fun onStop() { + lifecycleRegistry.currentState = Lifecycle.State.CREATED + super.onStop() + } override fun onDestroy() { + lifecycleRegistry.currentState = Lifecycle.State.DESTROYED super.onDestroy() // 清理资源 stopRecording() + try { interactionCoordinator.stop() } catch (_: Throwable) {} + try { sharedFaceCameraManager.release() } catch (_: Throwable) {} recordingJob?.cancel() asrWorkerJob?.cancel() ioScope.cancel() ttsController.stop() asrManager.release() + faceDetectionPipeline.release() llmManager?.destroy() instance = null } @@ -146,12 +221,26 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { grantResults: IntArray ) { super.onRequestPermissionsResult(requestCode, permissions, grantResults) - if (requestCode == AppConfig.REQUEST_RECORD_AUDIO_PERMISSION) { - if (grantResults.isNotEmpty() && grantResults[0] == PackageManager.PERMISSION_GRANTED) { - Log.d("UnityDigitalPerson", "麦克风权限已授予") - } else { - Log.e("UnityDigitalPerson", "麦克风权限被拒绝") - } + if (requestCode != AppConfig.REQUEST_RECORD_AUDIO_PERMISSION) return + if (grantResults.isEmpty()) { + finish() + return + } + val granted = permissions.zip(grantResults.toTypedArray()).associate { it.first to it.second } + val micGranted = granted[Manifest.permission.RECORD_AUDIO] == PackageManager.PERMISSION_GRANTED + val cameraGranted = granted[Manifest.permission.CAMERA] == PackageManager.PERMISSION_GRANTED + if (!micGranted) { + Log.e("UnityDigitalPerson", "麦克风权限被拒绝") + finish() + return + } + if (!cameraGranted) { + appendChat("[系统] 未授予相机权限，暂不启用主动打招呼。") + Log.w("UnityDigitalPerson", "Camera permission denied") + return + } + if (facePipelineReady) { + sharedFaceCameraManager.start() } } @@ -164,6 +253,7 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { // 获取UI组件 chatHistoryText = chatLayout.findViewById(R.id.my_text) holdToSpeakButton = chatLayout.findViewById(R.id.record_button) + recordButtonGlow = chatLayout.findViewById(R.id.record_button_glow) // 根据配置设置按钮可见性 if (AppConfig.USE_HOLD_TO_SPEAK) { @@ -175,8 +265,22 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { // 设置按钮监听器 holdToSpeakButton.setOnTouchListener { _, event -> when (event.action) { - MotionEvent.ACTION_DOWN -> onRecordButtonDown() - MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> onRecordButtonUp() + MotionEvent.ACTION_DOWN -> { + vibrateButton(true) + holdToSpeakButton.text = "松开结束" + holdToSpeakButton.backgroundTintList = ColorStateList.valueOf(Color.parseColor("#F44336")) + holdToSpeakButton.animate().scaleX(1.15f).scaleY(1.15f).setDuration(100).start() + startButtonPulseAnimation() + onRecordButtonDown() + } + MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> { + vibrateButton(false) + holdToSpeakButton.text = "按住说话" + holdToSpeakButton.backgroundTintList = ColorStateList.valueOf(Color.parseColor("#4CAF50")) + holdToSpeakButton.animate().scaleX(1.0f).scaleY(1.0f).setDuration(100).start() + stopButtonPulseAnimation() + onRecordButtonUp() + } } true } @@ -200,17 +304,31 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { // 内存模块 conversationBufferMemory = ConversationBufferMemory(database) userMemoryStore = UserMemoryStore(this) + // conversationSummaryMemory 接受 null llmManager，会自动降级使用云端LLM + conversationSummaryMemory = ConversationSummaryMemory(database, llmManager, CloudReflectionHelper.createCloudGenerator()) + + interactionCoordinator = createInteractionCoordinator() + cloudApiManager = CloudApiManager(createCloudApiListener(), applicationContext).apply { + setEnableStreaming(false) + } // 人脸检测 faceDetectionPipeline = FaceDetectionPipeline( context = this, onResult = { result -> - Log.d("UnityDigitalPerson", "Face detection result: ${result.faces.size} faces") +// Log.d("UnityDigitalPerson", "Face detection result: ${result.faces.size} faces") }, - onPresenceChanged = { present, faceIdentityId, recognizedName -> - Log.d("UnityDigitalPerson", "Presence changed: present=$present, faceId=$faceIdentityId, name=$recognizedName") + onPresenceChanged = { present, isFrontal, faceIdentityId, recognizedName -> +// Log.d("UnityDigitalPerson", "Presence changed: present=$present, isFrontal=$isFrontal, faceId=$faceIdentityId, name=$recognizedName") + interactionCoordinator.onFaceSignal(present, isFrontal, faceIdentityId, recognizedName) } ) + sharedFaceCameraManager = SharedFaceCameraManager( + context = applicationContext, + lifecycleOwner = this, + pipeline = faceDetectionPipeline, + previewView = null + ) // 音频处理器 audioProcessor = AudioProcessor(this) @@ -229,7 +347,6 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { Log.d("UnityDigitalPerson", "ASR result: $text") if (text.isNotEmpty()) { appendChat("用户: $text") - processUserMessage(text) } } @@ -237,12 +354,13 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { Log.d("UnityDigitalPerson", "ASR skipped: $reason") } - override fun shouldSkipAsr(): Boolean = false + override fun shouldSkipAsr(): Boolean = ttsController.isPlaying() - override fun isLlmInFlight(): Boolean = llmInFlight + override fun isLlmInFlight(): Boolean = interactionCoordinator.isLlmInFlight() override fun onLlmCalled(text: String) { Log.d("UnityDigitalPerson", "LLM called with: $text") + interactionCoordinator.onUserAsrText(text) } }) setAudioProcessor(audioProcessor) @@ -264,6 +382,7 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { override fun onTtsCompleted() { Log.d("UnityDigitalPerson", "TTS completed") stopTTSPlayback() + interactionCoordinator.onTtsPlaybackCompleted() } override fun onTtsSegmentCompleted(durationMs: Long) { @@ -291,13 +410,44 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { init() } - // 初始化LLM - initLLM() + // ⚠️ 本地LLM初始化已跳过（模型文件不存在） + // questionGenerationAgent 会在llmManager为null时自动降级使用云端LLM + + // 初始化题目生成智能体 + // 如果没有本地LLM，使用云端LLM + if (llmManager != null) { + // 使用本地LLM + val localAgent = QuestionGenerationAgent( + context = this, + llmManager = llmManager!!, + userMemoryStore = userMemoryStore + ) + questionGenerationAgent = localAgent + localAgent.start() + Log.i("UnityDigitalPerson", "QuestionGenerationAgent initialized with local LLM") + } else { + // 使用云端LLM - 创建一个简单的包装器 + Log.i("UnityDigitalPerson", "Initializing QuestionGenerationAgent with cloud LLM") + + val cloudAgent = QuestionGenerationAgent( + context = this, + llmManager = null, + userMemoryStore = userMemoryStore, + cloudLLMGenerator = CloudReflectionHelper.createCloudGenerator() + ) + questionGenerationAgent = cloudAgent + cloudAgent.start() + Log.i("UnityDigitalPerson", "QuestionGenerationAgent initialized with cloud LLM") + } // 初始化人脸检测 - faceDetectionPipeline.initialize() + facePipelineReady = faceDetectionPipeline.initialize() + if (!facePipelineReady) { + appendChat("[系统] 人脸管线初始化失败，主动打招呼不可用。") + } + interactionCoordinator.start() - // 检查权限并开始录音 + // 检查权限并启动相机/录音能力 checkPermissions() } @@ -330,48 +480,55 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { } val finalText = localLlmResponseBuffer.toString().trim() localLlmResponseBuffer.setLength(0) - if (finalText.isNotEmpty()) { - appendChat("助手: $finalText") - // 使用TTS播放回复 - ttsController.enqueueSegment(finalText) - ttsController.enqueueEnd() + val callback = pendingLocalThoughtCallback + pendingLocalThoughtCallback = null + if (callback != null) { + callback(finalText) + localThoughtSilentMode = false + return@runOnUiThread } - llmInFlight = false + if (!localThoughtSilentMode && finalText.isNotEmpty()) { + val filteredText = ttsController.speakLlmResponse(finalText) + Log.d("UnityDigitalPerson", "LOCAL filtered: ${filteredText?.take(60)}") + if (filteredText != null) appendChat("助手: $filteredText") + } + localThoughtSilentMode = false } } }) - - // 初始化ConversationSummaryMemory - conversationSummaryMemory = ConversationSummaryMemory( - AppDatabase.getInstance(this), - llmManager - ) - + } catch (e: Exception) { Log.e("UnityDigitalPerson", "Failed to initialize LLM: ${e.message}", e) + // Show user-friendly error + runOnUiThread { + android.widget.Toast.makeText( + this, + "LLM初始化失败: ${e.message}\n应用将继续运行，但部分功能不可用", + android.widget.Toast.LENGTH_LONG + ).show() + } } } // ==================== 权限检查 ==================== private fun checkPermissions() { - if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) - != PackageManager.PERMISSION_GRANTED) { + if (!allPermissionsGranted()) { ActivityCompat.requestPermissions( this, - micPermissions, + appPermissions, AppConfig.REQUEST_RECORD_AUDIO_PERMISSION ) + return + } + if (facePipelineReady) { + sharedFaceCameraManager.start() + } + } + + private fun allPermissionsGranted(): Boolean { + return appPermissions.all { + ContextCompat.checkSelfPermission(this, it) == PackageManager.PERMISSION_GRANTED } - - // 可选：检查摄像头权限 -// if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) -// != PackageManager.PERMISSION_GRANTED) { -// ActivityCompat.requestPermissions( -// this, -// cameraPermissions, -// AppConfig.REQUEST_CAMERA_PERMISSION -// ) -// } } // ==================== 录音控制 ==================== @@ -383,7 +540,6 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { return } - llmInFlight = false ttsController.reset() vadManager.reset() audioProcessor.startRecording() @@ -400,6 +556,7 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { private fun onRecordButtonDown() { if (isRecording) return ttsController.interruptForNewTurn() + interactionCoordinator.onUserStartSpeaking() holdToSpeakAudioBuffer.clear() startRecording() } @@ -496,17 +653,9 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { // ==================== 消息处理 ==================== private fun processUserMessage(message: String) { - conversationBufferMemory.addMessage(activeUserId, "user", message) - - llmInFlight = true - llmManager?.generateResponseWithSystem( - getSystemPrompt(), - message - ) - } - - private fun getSystemPrompt(): String { - return "你是一个友好的数字人助手。" + if (message.isBlank()) return + appendChat("用户: $message") + interactionCoordinator.onUserAsrText(message) } private fun appendChat(text: String) { @@ -514,10 +663,108 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { chatHistoryText.append(text + "\n") } } - - private val activeUserId: String - get() = "face_1" - + + private fun createInteractionCoordinator(): BaseDigitalPersonCoordinator { + return object : BaseDigitalPersonCoordinator( + scope = ioScope, + context = applicationContext, + userMemoryStore = userMemoryStore, + conversationBufferMemory = conversationBufferMemory, + conversationSummaryMemory = conversationSummaryMemory, + llmManager = llmManager, + cloudLLMGenerator = CloudReflectionHelper.createCloudGenerator(), + ) { + override fun onStateChangedInternal(state: InteractionState) { + super.onStateChangedInternal(state) // triggers analyzeUserProfileInIdleIfNeeded + appendChat("[State] $state") + sendMotionToUnity(mapStateToUnityTrigger(state)) + } + + override fun onProfileUpdated(userId: String) { + appendChat("[Memory] 已更新用户画像: $userId") + } + + override fun onAppendText(text: String) { + appendChat(text.trim()) + } + + override fun onSpeak(text: String) { + ttsController.enqueueSegment(text) + ttsController.enqueueEnd() + } + + override fun onRequestCloudReply(prompt: String) { + cloudApiManager.callLLM(prompt) + } + + override fun onRequestLocalThought(prompt: String, onResult: (String) -> Unit) { + this@UnityDigitalPersonActivity.requestLocalThought(prompt, onResult) + } + + override fun onAddConversation(role: String, content: String) { + val normalizedRole = if (role.equals("assistant", true) || role == "助手") "assistant" else "user" + conversationBufferMemory.addMessage(currentUserId(), normalizedRole, content) + } + + override fun onAddAssistantMessageToCloudHistory(content: String) { + cloudApiManager.addAssistantMessage(content) + } + + override fun onGetRandomQuestion(faceId: String): String { + val question = userMemoryStore.getRandomUnansweredQuestion(faceId) + return question?.content ?: "你喜欢做什么呀？" + } + + override fun onQuestionAskedCallback(userId: String) { + // 每次提问后触发题目生成检查 + questionGenerationAgent?.onQuestionAsked(userId) + ?: Log.w(TAG, "QuestionGenerationAgent not initialized, skipping replenishment") + } + + override fun onFaceAppearedCallback(userId: String) { + // 人脸出现时立即触发题目预生成检查 + Log.i(TAG, "Face appeared, triggering question pre-generation for user: $userId") + questionGenerationAgent?.onQuestionAsked(userId) + ?: Log.w(TAG, "QuestionGenerationAgent not initialized yet, skipping question generation") + } + } + } + + private fun createCloudApiListener() = object : CloudApiManager.CloudApiListener { + override fun onLLMResponseReceived(response: String) { + val filteredText = ttsController.speakLlmResponse(response) + android.util.Log.d("UnityDigitalPerson", "LLM response filtered: ${filteredText?.take(60)}") + if (filteredText != null) appendChat("助手: $filteredText") + interactionCoordinator.onCloudFinalResponse(filteredText ?: response.trim()) + } + + override fun onLLMStreamingChunkReceived(chunk: String) { + // Unity 页面当前固定非流式，这里保留空实现 + } + + override fun onTTSAudioReceived(audioFilePath: String) {} + + override fun onError(errorMessage: String) { + appendChat("[错误] $errorMessage") + Toast.makeText(this@UnityDigitalPersonActivity, errorMessage, Toast.LENGTH_LONG).show() + interactionCoordinator.onCloudError() + } + } + + private fun requestLocalThought(prompt: String, onResult: (String) -> Unit) { + val local = llmManager + if (local == null) { + onResult("我在想，下次见面聊点更有趣的话题。") + return + } + localThoughtSilentMode = true + pendingLocalThoughtCallback = onResult + local.generateResponseWithSystem( + "你是数字人内心独白模块，输出一句简短温和的想法。", + prompt + ) + } + // ==================== TTS控制 ==================== private fun startTTSPlayback() { if (isTTSPlaying) return @@ -555,4 +802,61 @@ class UnityDigitalPersonActivity : UnityPlayerActivity() { } ttsHandler.postDelayed(ttsStopRunnable!!, 500) // 500ms延迟，避免短暂中断 } + + private fun sendMotionToUnity(trigger: String) { + try { + UnityPlayer.UnitySendMessage(unityAudioTargetObject, "OnPlayMotion", trigger) + Log.d("UnityDigitalPerson", "sendMotionToUnity: $trigger") + } catch (e: Exception) { + Log.w("UnityDigitalPerson", "sendMotionToUnity failed: ${e.message}") + } + } + + private fun mapStateToUnityTrigger(state: InteractionState): String { + return when (state) { + InteractionState.IDLE -> "idle" + InteractionState.MEMORY -> "idle" + InteractionState.GREETING -> "waving" + InteractionState.WAITING_REPLY -> "listen" + InteractionState.DIALOGUE -> "talking" + InteractionState.PROACTIVE -> "ack" + InteractionState.FAREWELL -> "bye" + } + } + + // ==================== 按钮反馈辅助方法 ==================== + private fun vibrateButton(isPress: Boolean) { + val vibrator = getSystemService(Context.VIBRATOR_SERVICE) as? Vibrator + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val effect = if (isPress) + VibrationEffect.createPredefined(VibrationEffect.EFFECT_CLICK) + else + VibrationEffect.createPredefined(VibrationEffect.EFFECT_TICK) + vibrator?.vibrate(effect) + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val duration = if (isPress) 30L else 15L + vibrator?.vibrate(VibrationEffect.createOneShot(duration, VibrationEffect.DEFAULT_AMPLITUDE)) + } else { + @Suppress("DEPRECATION") + vibrator?.vibrate(if (isPress) 30L else 15L) + } + } + + private fun startButtonPulseAnimation() { + val glow = recordButtonGlow ?: return + glow.visibility = View.VISIBLE + glow.alpha = 0.6f + pulseAnimator = ObjectAnimator.ofFloat(glow, "alpha", 0.6f, 0.2f).apply { + duration = 800 + repeatMode = ObjectAnimator.REVERSE + repeatCount = ObjectAnimator.INFINITE + start() + } + } + + private fun stopButtonPulseAnimation() { + pulseAnimator?.cancel() + pulseAnimator = null + recordButtonGlow?.visibility = View.GONE + } } \ No newline at end of file diff --git a/app/src/main/java/com/digitalperson/cloud/CloudReflectionHelper.kt b/app/src/main/java/com/digitalperson/cloud/CloudReflectionHelper.kt new file mode 100644 index 0000000..a3053df --- /dev/null +++ b/app/src/main/java/com/digitalperson/cloud/CloudReflectionHelper.kt @@ -0,0 +1,213 @@ +package com.digitalperson.cloud + +import android.util.Log +import com.digitalperson.BuildConfig +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.json.JSONObject +import java.net.HttpURLConnection +import java.net.URL + +/** + * Shared helper for cloud LLM reflection tasks: + * - User profile extraction from conversations + * - Direct (non-streaming) cloud LLM calls + */ +object CloudReflectionHelper { + private const val TAG = "CloudReflectionHelper" + + /** + * Call cloud LLM directly (non-streaming, synchronous within coroutine). + * Must be called from a coroutine (suspend function). + */ + suspend fun callCloudLLM(prompt: String): String = withContext(Dispatchers.IO) { + try { + val url = URL(BuildConfig.LLM_API_URL) + val conn = url.openConnection() as HttpURLConnection + conn.requestMethod = "POST" + conn.setRequestProperty("Content-Type", "application/json") + conn.setRequestProperty("Authorization", "Bearer ${BuildConfig.LLM_API_KEY}") + conn.doOutput = true + conn.connectTimeout = 10000 + conn.readTimeout = 60000 + + val requestBody = JSONObject() + requestBody.put("model", BuildConfig.LLM_MODEL) + val messages = org.json.JSONArray() + val userMessage = JSONObject() + userMessage.put("role", "user") + userMessage.put("content", prompt) + messages.put(userMessage) + requestBody.put("messages", messages) + requestBody.put("stream", false) + + val outputStream = conn.outputStream + outputStream.write(requestBody.toString().toByteArray()) + outputStream.close() + + val responseCode = conn.responseCode + if (responseCode == HttpURLConnection.HTTP_OK) { + val response = conn.inputStream.bufferedReader().use { it.readText() } + val responseJson = JSONObject(response) + val choices = responseJson.getJSONArray("choices") + if (choices.length() > 0) { + val choice = choices.getJSONObject(0) + val message = choice.getJSONObject("message") + val content = message.getString("content") + Log.d(TAG, "Cloud LLM response: ${content.take(100)}...") + content + } else { + Log.e(TAG, "No choices in cloud LLM response") + "" + } + } else { + Log.e(TAG, "Cloud LLM error: $responseCode") + "" + } + } catch (e: Exception) { + Log.e(TAG, "Failed to call cloud LLM", e) + "" + } + } + + /** + * Extract user profile from dialogue using a single structured cloud LLM call. + * Returns a map with keys: name, age, gender, hobbies, summary. + * Runs asynchronously; result delivered via onResult callback. + */ + fun extractUserProfile(dialogue: String, onResult: (Map) -> Unit) { + Log.d(TAG, "Extracting user profile with cloud LLM...") + CoroutineScope(Dispatchers.IO).launch { + try { + val prompt = """ +请根据以下对话，提取用户的个人信息。请以JSON格式返回，包含以下字段： +- name: 用户姓名（如果没有提到，返回"未知"） +- age: 用户年龄（如果没有提到，返回"未知"） +- gender: 用户性别（如果没有提到，返回"未知"） +- hobbies: 用户爱好（如果没有提到，返回"未知"） +- summary: 对话内容的简要总结 + +只返回JSON，不要有其他内容。示例格式： +{"name":"张三","age":"8岁","gender":"男","hobbies":"画画、唱歌","summary":"用户询问了关于画画的技巧"} + +对话内容： +$dialogue + """.trimIndent() + + val answer = callCloudLLM(prompt) + val results = mutableMapOf() + + try { + val jsonStr = answer.trim().let { raw -> + if (raw.startsWith("```")) { + raw.substringAfter("{").substringBeforeLast("}").let { "{$it}" } + } else { + raw + } + } + val json = JSONObject(jsonStr) + + results["name"] = processProfileAnswer(json.optString("name", "未知")) + results["age"] = processProfileAnswer(json.optString("age", "未知")) + results["gender"] = processProfileAnswer(json.optString("gender", "未知")) + results["hobbies"] = processProfileAnswer(json.optString("hobbies", "未知")) + results["summary"] = processProfileAnswer(json.optString("summary", "未知")) + } catch (e: Exception) { + Log.w(TAG, "Failed to parse profile JSON, attempting fallback: ${e.message}") + results["summary"] = processProfileAnswer(answer) + } + + onResult(results) + } catch (e: Exception) { + Log.e(TAG, "Cloud profile extraction failed: ${e.message}", e) + onResult(emptyMap()) + } + } + } + + /** + * Process a profile answer: filter out "未知"/unknown values, clean up formatting. + */ + fun processProfileAnswer(answer: String): String { + var processed = answer.replace("<", "").replace(">", "") + if (processed.contains("unknown", ignoreCase = true) || + processed.contains("null", ignoreCase = true) || + processed.contains("未知")) { + return "" + } + if (processed.contains(":")) { + processed = processed.substringAfter(":").trim() + } + processed = processed.replace(".", "").trim() + return processed + } + + /** + * Extract user profile using local LLM with multi-question approach. + * Local models are less capable, so we ask 5 separate simple questions. + */ + fun extractUserProfileWithLocalLLM( + dialogue: String, + llmGenerate: (String, (String) -> Unit) -> Unit, + onResult: (Map) -> Unit + ) { + val questions = listOf( + "请从对话中提取用户的姓名，只返回姓名，如果没有提到姓名，请返回未知", + "请从对话中提取用户的年龄，只返回年龄，如果没有提到年龄，请返回未知", + "请从对话中提取用户的性别，只返回性别，如果没有提到性别，请返回未知", + "请从对话中提取用户的爱好，只返回爱好，如果没有提到爱好，请返回未知", + "请总结对话，只返回总结的内容" + ) + + var completed = 0 + val results = mutableMapOf() + + questions.forEach { question -> + val prompt = """ +请根据以下对话回答问题： + +对话内容： +$dialogue + +问题：$question + +回答： + """.trimIndent() + + llmGenerate(prompt) { answer -> + val processed = processProfileAnswer(answer) + when { + question.contains("姓名") -> results["name"] = processed + question.contains("年龄") -> results["age"] = processed + question.contains("性别") -> results["gender"] = processed + question.contains("爱好") -> results["hobbies"] = processed + question.contains("总结") -> results["summary"] = processed + } + + synchronized(results) { + completed++ + if (completed == questions.size) { + onResult(results) + } + } + } + } + } + + /** + * Create a cloud LLM generator lambda suitable for passing to ConversationSummaryMemory. + */ + fun createCloudGenerator(): (String, (String) -> Unit) -> Unit = { prompt, callback -> + CoroutineScope(Dispatchers.IO).launch { + try { + val result = callCloudLLM(prompt) + callback(result) + } catch (e: Exception) { + Log.e(TAG, "Cloud LLM generator failed", e) + callback("") + } + } + } +} diff --git a/app/src/main/java/com/digitalperson/data/dao/QuestionDao.kt b/app/src/main/java/com/digitalperson/data/dao/QuestionDao.kt index 2cec4f2..9dfa4a1 100644 --- a/app/src/main/java/com/digitalperson/data/dao/QuestionDao.kt +++ b/app/src/main/java/com/digitalperson/data/dao/QuestionDao.kt @@ -41,4 +41,27 @@ interface QuestionDao { @Query("SELECT DISTINCT subject FROM questions WHERE subject IS NOT NULL ORDER BY subject") fun getAllSubjects(): List + + @Query(""" + SELECT COUNT(*) + FROM questions q + WHERE NOT EXISTS ( + SELECT 1 FROM user_answers ua + WHERE ua.question_id = q.id AND ua.user_id = :userId + ) + """) + fun countUnansweredQuestions(userId: String): Int + + @Query(""" + SELECT q.* + FROM questions q + WHERE NOT EXISTS ( + SELECT 1 FROM user_answers ua + WHERE ua.question_id = q.id AND ua.user_id = :userId + ) + AND (:subject IS NULL OR q.subject = :subject) + AND (:grade IS NULL OR q.grade = :grade) + ORDER BY RANDOM() LIMIT :limit + """) + fun getRandomUnansweredQuestions(userId: String, subject: String?, grade: Int?, limit: Int): List } \ No newline at end of file diff --git a/app/src/main/java/com/digitalperson/face/FaceDetectionPipeline.kt b/app/src/main/java/com/digitalperson/face/FaceDetectionPipeline.kt index 73888c5..2511c28 100644 --- a/app/src/main/java/com/digitalperson/face/FaceDetectionPipeline.kt +++ b/app/src/main/java/com/digitalperson/face/FaceDetectionPipeline.kt @@ -33,12 +33,11 @@ data class FaceDetectionResult( class FaceDetectionPipeline( context: Context, private val onResult: (FaceDetectionResult) -> Unit, - private val onPresenceChanged: (present: Boolean, faceIdentityId: String?, recognizedName: String?) -> Unit, + private val onPresenceChanged: (present: Boolean, isFrontal: Boolean, faceIdentityId: String?, recognizedName: String?) -> Unit, ) { private val appContext = context.applicationContext private val engine = RetinaFaceEngineRKNN() private val recognizer = FaceRecognizer(appContext) - private val userMemoryStore = com.digitalperson.interaction.UserMemoryStore(appContext) private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default) private val frameInFlight = AtomicBoolean(false) private val initialized = AtomicBoolean(false) @@ -76,7 +75,7 @@ class FaceDetectionPipeline( val raw = engine.detect(bitmap) if (raw.isEmpty()) { withContext(Dispatchers.Main) { - onPresenceChanged(false, null, null) + onPresenceChanged(false, false, null, null) onResult(FaceDetectionResult(width, height, emptyList())) } return@launch @@ -124,8 +123,15 @@ class FaceDetectionPipeline( // } maybeRecognize(bitmap, filteredFaces) + // 判断是否为正脸 + val primaryFace = filteredFaces.maxByOrNull { (it.right - it.left) * (it.bottom - it.top) } + val isFrontal = if (primaryFace != null) { + isFrontal(primaryFace, bitmap.width, bitmap.height) + } else { + false + } withContext(Dispatchers.Main) { - onPresenceChanged(filteredFaces.isNotEmpty(), lastFaceIdentityId, lastRecognizedName) + onPresenceChanged(filteredFaces.isNotEmpty(), isFrontal, lastFaceIdentityId, lastRecognizedName) onResult(FaceDetectionResult(width, height, filteredFaces)) } } catch (t: Throwable) { @@ -182,13 +188,11 @@ class FaceDetectionPipeline( val match = recognizer.resolveIdentityFromEmbedding(fused) analyzedTrackId = trackId lastFaceIdentityId = match.matchedId?.let { "face_$it" } - // 从 user_memory 表中获取名字 - lastRecognizedName = lastFaceIdentityId?.let { userId -> - userMemoryStore.getMemory(userId)?.displayName - } + // Name resolution is handled by the coordinator's UserMemoryStore, not here + lastRecognizedName = null Log.i( AppConfig.TAG, - "[Face] stable track=$trackId faceId=${lastFaceIdentityId} matched=${lastRecognizedName} score=${match.similarity} fusionN=${fusionEmbeddings.size}" + "[Face] stable track=$trackId faceId=${lastFaceIdentityId} score=${match.similarity} fusionN=${fusionEmbeddings.size}" ) } diff --git a/app/src/main/java/com/digitalperson/face/SharedFaceCameraManager.kt b/app/src/main/java/com/digitalperson/face/SharedFaceCameraManager.kt new file mode 100644 index 0000000..1664ded --- /dev/null +++ b/app/src/main/java/com/digitalperson/face/SharedFaceCameraManager.kt @@ -0,0 +1,86 @@ +package com.digitalperson.face + +import android.content.Context +import android.util.Log +import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.Preview +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.view.PreviewView +import androidx.core.content.ContextCompat +import androidx.lifecycle.LifecycleOwner +import com.digitalperson.config.AppConfig +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +class SharedFaceCameraManager( + context: Context, + private val lifecycleOwner: LifecycleOwner, + private val pipeline: FaceDetectionPipeline, + private val previewView: PreviewView? = null, +) { + private val appContext = context.applicationContext + private val analyzerExecutor: ExecutorService = Executors.newSingleThreadExecutor() + private var cameraProvider: ProcessCameraProvider? = null + + fun start() { + val cameraProviderFuture = ProcessCameraProvider.getInstance(appContext) + cameraProviderFuture.addListener({ + try { + val provider = cameraProviderFuture.get() + cameraProvider = provider + provider.unbindAll() + + val analyzer = ImageAnalysis.Builder() + .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) + .build() + analyzer.setAnalyzer(analyzerExecutor) { imageProxy -> + try { + val bitmap = ImageProxyBitmapConverter.toBitmap(imageProxy) + if (bitmap != null) { + pipeline.submitFrame(bitmap) + } + } catch (t: Throwable) { + Log.w(AppConfig.TAG, "Face camera analyze failed: ${t.message}") + } finally { + imageProxy.close() + } + } + + val selector = CameraSelector.Builder() + .requireLensFacing(CameraSelector.LENS_FACING_FRONT) + .build() + + if (previewView != null) { + previewView.implementationMode = PreviewView.ImplementationMode.COMPATIBLE + previewView.scaleType = PreviewView.ScaleType.FIT_CENTER + val preview = Preview.Builder().build().apply { + setSurfaceProvider(previewView.surfaceProvider) + } + provider.bindToLifecycle(lifecycleOwner, selector, preview, analyzer) + } else { + provider.bindToLifecycle(lifecycleOwner, selector, analyzer) + } + } catch (t: Throwable) { + Log.e(AppConfig.TAG, "Face camera start failed: ${t.message}", t) + } + }, ContextCompat.getMainExecutor(appContext)) + } + + fun stop() { + try { + cameraProvider?.unbindAll() + } catch (_: Throwable) { + } finally { + cameraProvider = null + } + } + + fun release() { + stop() + try { + analyzerExecutor.shutdown() + } catch (_: Throwable) { + } + } +} diff --git a/app/src/main/java/com/digitalperson/interaction/BaseDigitalPersonCoordinator.kt b/app/src/main/java/com/digitalperson/interaction/BaseDigitalPersonCoordinator.kt new file mode 100644 index 0000000..a253928 --- /dev/null +++ b/app/src/main/java/com/digitalperson/interaction/BaseDigitalPersonCoordinator.kt @@ -0,0 +1,270 @@ +package com.digitalperson.interaction + +import android.content.Context +import android.util.Log +import com.digitalperson.cloud.CloudReflectionHelper +import com.digitalperson.llm.LLMManager + +abstract class BaseDigitalPersonCoordinator( + private val scope: kotlinx.coroutines.CoroutineScope, + private val context: Context, + private val userMemoryStore: UserMemoryStore, + private val conversationBufferMemory: ConversationBufferMemory, + private val conversationSummaryMemory: ConversationSummaryMemory, + private val llmManager: LLMManager? = null, + private val cloudLLMGenerator: ((String, (String) -> Unit) -> Unit)? = null, +) { + private val TAG_PROFILE = "BaseCoordinator" + private var activeUserId: String = "guest" + private var lastFacePresent: Boolean = false + private var lastFaceIdentityId: String? = null + private var lastFaceRecognizedName: String? = null + private var llmInFlight: Boolean = false + // True only after a cloud dialogue reply (or error) arrives, cleared once TTS completes. + // Ensures onTtsPlaybackCompleted() only advances the state machine for dialogue turns, + // not for greeting / farewell / proactive TTS. + private var pendingDialogueFinish: Boolean = false + + private val controller = DigitalHumanInteractionController( + scope = scope, + handler = object : InteractionActionHandler { + override fun onStateChanged(state: InteractionState) { + onStateChangedInternal(state) + } + + override fun appendText(text: String) { + onAppendText(text) + } + + override fun speak(text: String) { + onSpeak(text) + } + + override fun requestCloudReply(userText: String) { + setLlmInFlight(true) + onRequestCloudReply(buildCloudPromptWithUserProfile(userText)) + } + + override fun requestLocalThought(prompt: String, onResult: (String) -> Unit) { + onRequestLocalThought(prompt, onResult) + } + + override fun onRememberUser(faceIdentityId: String, name: String?) { + activeUserId = faceIdentityId + userMemoryStore.upsertUserSeen(activeUserId, name) + onActiveUserChanged(activeUserId, name) + } + + override fun saveThought(thought: String) { + userMemoryStore.upsertUserSeen(activeUserId, null) + userMemoryStore.updateThought(activeUserId, thought) + } + + override fun loadLatestThought(): String? = userMemoryStore.getLatestThought() + + override fun loadRecentThoughts(timeRangeMs: Long): List = userMemoryStore.getRecentThoughts(timeRangeMs) + + override fun addToChatHistory(role: String, content: String) { + onAddConversation(role, content) + } + + override fun addAssistantMessageToCloudHistory(content: String) { + onAddAssistantMessageToCloudHistory(content) + } + + override fun getRandomQuestion(faceId: String): String { + return onGetRandomQuestion(faceId) + } + + override fun onQuestionAsked(userId: String) { + onQuestionAskedCallback(userId) + } + + override fun onFaceAppeared(userId: String) { + onFaceAppearedCallback(userId) + } + + override fun getStoredDisplayName(faceId: String): String? { + return userMemoryStore.getMemory(faceId)?.displayName?.takeIf { it.isNotBlank() } + } + }, + context = context + ) + + fun start() { + controller.start() + } + + fun stop() { + controller.stop() + } + + fun onFaceSignal(present: Boolean, isFrontal: Boolean, faceIdentityId: String?, recognizedName: String?) { + if (present != lastFacePresent) { + lastFacePresent = present + controller.onFacePresenceChanged(present, isFrontal) // 传递 isFrontal 参数 + if (!present) { + lastFaceIdentityId = null + lastFaceRecognizedName = null + } + } + if (present && (faceIdentityId != lastFaceIdentityId || recognizedName != lastFaceRecognizedName)) { + lastFaceIdentityId = faceIdentityId + lastFaceRecognizedName = recognizedName + controller.onFaceIdentityUpdated(faceIdentityId, recognizedName) + } + } + + fun onUserStartSpeaking() { + controller.onUserStartSpeaking() + } + + fun onUserAsrText(text: String) { + if (text.isBlank()) return + onAddConversation("user", text) + controller.onUserAsrText(text) + } + + fun onCloudFinalResponse(response: String) { + setLlmInFlight(false) + if (response.isNotBlank()) { + onAddConversation("assistant", response) + } + // Mark that a dialogue reply is pending TTS completion. + // onTtsPlaybackCompleted() will advance the state machine once TTS finishes. + pendingDialogueFinish = true + } + + fun onCloudError() { + setLlmInFlight(false) + // On error there is no TTS, so advance the state immediately. + pendingDialogueFinish = false + controller.onDialogueResponseFinished() + } + + /** + * Call this when TTS playback of the assistant's turn has fully completed. + * Only advances the state machine when a dialogue reply was pending + * (i.e. after a cloud LLM response), NOT after greeting / farewell / proactive TTS. + */ + fun onTtsPlaybackCompleted() { + if (pendingDialogueFinish) { + pendingDialogueFinish = false + controller.onDialogueResponseFinished() + } + } + + fun isLlmInFlight(): Boolean = llmInFlight + + fun currentUserId(): String = activeUserId + + protected open fun onStateChangedInternal(state: InteractionState) { + if (state == InteractionState.IDLE) { + analyzeUserProfileInIdleIfNeeded() + } + } + + protected open fun onActiveUserChanged(faceId: String, recognizedName: String?) {} + + protected abstract fun onGetRandomQuestion(faceId: String): String + + protected open fun buildCloudPromptWithUserProfile(userText: String): String { + val profile = userMemoryStore.getMemory(activeUserId) ?: return userText + val profileParts = ArrayList() + profile.displayName?.takeIf { it.isNotBlank() }?.let { profileParts.add("姓名:$it") } + profile.age?.takeIf { it.isNotBlank() }?.let { profileParts.add("年龄:$it") } + profile.gender?.takeIf { it.isNotBlank() }?.let { profileParts.add("性别:$it") } + profile.hobbies?.takeIf { it.isNotBlank() }?.let { profileParts.add("爱好:$it") } + profile.profileSummary?.takeIf { it.isNotBlank() }?.let { profileParts.add("画像:$it") } + if (profileParts.isEmpty()) return userText + return buildString { + append("[用户画像]\n") + append(profileParts.joinToString("；")) + append("\n[/用户画像]\n") + append(userText) + } + } + + private fun setLlmInFlight(inFlight: Boolean) { + if (llmInFlight == inFlight) return + llmInFlight = inFlight + onLlmInFlightChanged(inFlight) + } + + protected open fun onLlmInFlightChanged(inFlight: Boolean) {} + + protected abstract fun onAppendText(text: String) + protected abstract fun onSpeak(text: String) + protected abstract fun onRequestCloudReply(prompt: String) + protected abstract fun onRequestLocalThought(prompt: String, onResult: (String) -> Unit) + protected abstract fun onAddConversation(role: String, content: String) + protected abstract fun onAddAssistantMessageToCloudHistory(content: String) + + // 问题被问后的回调，用于触发题目生成检查 + protected open fun onQuestionAskedCallback(userId: String) { + // 子类可以覆写此方法来实现题目生成逻辑 + } + + // 人脸出现时的回调，用于预生成题目 + protected open fun onFaceAppearedCallback(userId: String) { + // 子类可以覆写此方法来实现题目预生成逻辑 + } + + // 用户画像更新回调，子类可以覆写以显示 UI 提示 + protected open fun onProfileUpdated(userId: String) {} + + private fun analyzeUserProfileInIdleIfNeeded() { + if (!activeUserId.startsWith("face_")) { + Log.d(TAG_PROFILE, "Skip profile analysis: faceID is not face_ (activeUserId=$activeUserId)") + return + } + val messages = conversationBufferMemory.getMessages(activeUserId) + Log.d(TAG_PROFILE, "Profile analysis: messages empty? ${messages.isEmpty()}") + val hasUserMessages = messages.any { it.role == "user" } + Log.d(TAG_PROFILE, "Profile analysis: has user messages? $hasUserMessages") + if (messages.isEmpty() || !hasUserMessages) return + + // 生成对话摘要 + conversationSummaryMemory.generateSummary(activeUserId, messages) { summary -> + Log.d(TAG_PROFILE, "Generated conversation summary for $activeUserId: $summary") + } + + // 提取用户画像 + val dialogue = messages.joinToString("\n") { "${it.role}: ${it.content}" } + requestMultiAngleProfileExtraction(dialogue) { profileData -> + try { + val nameToUpdate = profileData["name"]?.trim()?.ifBlank { null } + val ageToUpdate = profileData["age"]?.trim()?.ifBlank { null } + val genderToUpdate = profileData["gender"]?.trim()?.ifBlank { null } + val hobbiesToUpdate = profileData["hobbies"]?.trim()?.ifBlank { null } + val summaryToUpdate = profileData["summary"]?.trim()?.ifBlank { null } + Log.d(TAG_PROFILE, "profileData: $profileData") + if (nameToUpdate != null || ageToUpdate != null || genderToUpdate != null + || hobbiesToUpdate != null || summaryToUpdate != null) { + if (nameToUpdate != null) { + userMemoryStore.updateDisplayName(activeUserId, nameToUpdate) + Log.i(TAG_PROFILE, "Updated display name to $nameToUpdate") + } + userMemoryStore.updateProfile(activeUserId, ageToUpdate, genderToUpdate, hobbiesToUpdate, summaryToUpdate) + conversationBufferMemory.clear(activeUserId) + onProfileUpdated(activeUserId) + } + } catch (e: Exception) { + Log.w(TAG_PROFILE, "Profile parse failed: ${e.message}") + } + } + } + + private fun requestMultiAngleProfileExtraction(dialogue: String, onResult: (Map) -> Unit) { + val local = llmManager + if (local != null) { + CloudReflectionHelper.extractUserProfileWithLocalLLM( + dialogue = dialogue, + llmGenerate = { prompt, callback -> local.generate(prompt, callback) }, + onResult = onResult + ) + } else { + CloudReflectionHelper.extractUserProfile(dialogue, onResult) + } + } +} diff --git a/app/src/main/java/com/digitalperson/interaction/ConversationMemory.kt b/app/src/main/java/com/digitalperson/interaction/ConversationMemory.kt index 1dfe12d..1c7ee0c 100644 --- a/app/src/main/java/com/digitalperson/interaction/ConversationMemory.kt +++ b/app/src/main/java/com/digitalperson/interaction/ConversationMemory.kt @@ -147,7 +147,11 @@ class ConversationBufferMemory(private val db: AppDatabase, private val maxMessa * 对话摘要内存管理类 * 用于生成和存储对话摘要 */ -class ConversationSummaryMemory(private val db: AppDatabase, private val llmManager: LLMManager?) { +class ConversationSummaryMemory( + private val db: AppDatabase, + private val llmManager: LLMManager?, + private val cloudLLMGenerator: ((String, (String) -> Unit) -> Unit)? = null +) { private val userSummaries = mutableMapOf() private val TAG = "ConversationSummaryMemory" @@ -172,8 +176,8 @@ class ConversationSummaryMemory(private val db: AppDatabase, private val llmMana return } - if (llmManager == null) { - Log.w(TAG, "LLM manager is not initialized, cannot generate summary") + if (llmManager == null && cloudLLMGenerator == null) { + Log.w(TAG, "No LLM available for summary generation") onComplete("") return } @@ -195,30 +199,57 @@ class ConversationSummaryMemory(private val db: AppDatabase, private val llmMana """.trimIndent() Log.d(TAG, "Generating summary for user $userId") - llmManager.generate(prompt) { summary -> - val trimmedSummary = summary.trim() - userSummaries[userId] = trimmedSummary - - // 保存到数据库 - GlobalScope.launch(Dispatchers.IO) { - try { - db.conversationSummaryDao().clearSummariesByUserId(userId) - db.conversationSummaryDao().insert( - ConversationSummaryEntity( - id = 0, - userId = userId, - summary = trimmedSummary, - createdAt = System.currentTimeMillis() + if (llmManager != null) { + llmManager.generate(prompt) { summary -> + val trimmedSummary = summary.trim() + userSummaries[userId] = trimmedSummary + + // 保存到数据库 + GlobalScope.launch(Dispatchers.IO) { + try { + db.conversationSummaryDao().clearSummariesByUserId(userId) + db.conversationSummaryDao().insert( + ConversationSummaryEntity( + id = 0, + userId = userId, + summary = trimmedSummary, + createdAt = System.currentTimeMillis() + ) ) - ) - Log.d(TAG, "Saved summary for user $userId to database") - } catch (e: Exception) { - Log.e(TAG, "Error saving summary to database: ${e.message}") + Log.d(TAG, "Saved summary for user $userId to database") + } catch (e: Exception) { + Log.e(TAG, "Error saving summary to database: ${e.message}") + } } + + onComplete(trimmedSummary) + Log.d(TAG, "Generated summary for user $userId: $trimmedSummary") + } + } else { + cloudLLMGenerator?.invoke(prompt) { summary -> + val trimmedSummary = summary.trim() + userSummaries[userId] = trimmedSummary + + GlobalScope.launch(Dispatchers.IO) { + try { + db.conversationSummaryDao().clearSummariesByUserId(userId) + db.conversationSummaryDao().insert( + ConversationSummaryEntity( + id = 0, + userId = userId, + summary = trimmedSummary, + createdAt = System.currentTimeMillis() + ) + ) + Log.d(TAG, "Saved summary for user $userId to database") + } catch (e: Exception) { + Log.e(TAG, "Error saving summary to database: ${e.message}") + } + } + + onComplete(trimmedSummary) + Log.d(TAG, "Generated summary for user $userId: $trimmedSummary") } - - onComplete(trimmedSummary) - Log.d(TAG, "Generated summary for user $userId: $trimmedSummary") } } diff --git a/app/src/main/java/com/digitalperson/interaction/DigitalHumanInteractionController.kt b/app/src/main/java/com/digitalperson/interaction/DigitalHumanInteractionController.kt index 789cf0f..27a495f 100644 --- a/app/src/main/java/com/digitalperson/interaction/DigitalHumanInteractionController.kt +++ b/app/src/main/java/com/digitalperson/interaction/DigitalHumanInteractionController.kt @@ -26,7 +26,6 @@ enum class LlmRoute { interface InteractionActionHandler { fun onStateChanged(state: InteractionState) - fun playMotion(motionName: String) fun appendText(text: String) fun speak(text: String) fun requestCloudReply(userText: String) @@ -38,6 +37,9 @@ interface InteractionActionHandler { fun addToChatHistory(role: String, content: String) fun addAssistantMessageToCloudHistory(content: String) fun getRandomQuestion(faceId: String): String + fun onQuestionAsked(userId: String) // 问题被问后的回调 + fun onFaceAppeared(userId: String) // 人脸出现时的回调（用于预生成题目） + fun getStoredDisplayName(faceId: String): String? // 从DB获取已保存的用户名 } class DigitalHumanInteractionController( @@ -67,8 +69,8 @@ class DigitalHumanInteractionController( scheduleMemoryMode() } -fun onFacePresenceChanged(present: Boolean) { - Log.d(TAG, "onFacePresenceChanged: present=$present, state=$state") +fun onFacePresenceChanged(present: Boolean, isFrontal: Boolean = true) { // 添加默认参数保持兼容性 + Log.d(TAG, "onFacePresenceChanged: present=$present, isFrontal=$isFrontal, state=$state") facePresent = present val now = System.currentTimeMillis() @@ -80,40 +82,64 @@ fun onFacePresenceChanged(present: Boolean) { faceSeenSinceMs = 0L } - // 首次出现就启动稳定计时，到点只要人还在就问候。 - if (present) { - val stableMs = now - faceSeenSinceMs - val remain = AppConfig.Face.STABLE_MS - stableMs + // 只有在正脸的情况下才启动打招呼逻辑 + if (present && isFrontal) { + val stableMs = now - faceSeenSinceMs + val remain = AppConfig.Face.STABLE_MS - stableMs - if (remain > 0) { - faceStableJob?.cancel() - faceStableJob = scope.launch { - delay(remain) - if (!facePresent) return@launch - if (state == InteractionState.IDLE || state == InteractionState.MEMORY || state == InteractionState.FAREWELL) { - if (currentFaceId.isNullOrBlank()) { - Log.d(TAG, "Greeting as unknown user: identity still unavailable after stable timeout") + if (remain > 0) { + faceStableJob?.cancel() + faceStableJob = scope.launch { + delay(remain) + if (!facePresent) return@launch + if (state == InteractionState.IDLE || state == InteractionState.MEMORY || state == InteractionState.FAREWELL) { + // Wait for face recognition to complete before greeting (max 1500ms extra) + val recognitionWaitMs = 1500L + val pollIntervalMs = 100L + var waited = 0L + while (currentFaceId.isNullOrBlank() && waited < recognitionWaitMs && facePresent) { + delay(pollIntervalMs) + waited += pollIntervalMs + } + if (!facePresent) return@launch + Log.d(TAG, "Greeting after ${waited}ms recognition wait, faceId=$currentFaceId, name=$recognizedName") + handler.onFaceAppeared(currentFaceId ?: "guest") + enterGreeting() } + } + } else if (state == InteractionState.IDLE || state == InteractionState.MEMORY || state == InteractionState.FAREWELL) { + // Already stable — still wait for recognition + faceStableJob?.cancel() + faceStableJob = scope.launch { + val recognitionWaitMs = 1500L + val pollIntervalMs = 100L + var waited = 0L + while (currentFaceId.isNullOrBlank() && waited < recognitionWaitMs && facePresent) { + delay(pollIntervalMs) + waited += pollIntervalMs + } + if (!facePresent) return@launch + Log.d(TAG, "Greeting (immediate) after ${waited}ms recognition wait, faceId=$currentFaceId, name=$recognizedName") + handler.onFaceAppeared(currentFaceId ?: "guest") enterGreeting() } } - } else if (state == InteractionState.IDLE || state == InteractionState.MEMORY || state == InteractionState.FAREWELL) { - enterGreeting() + return } - return - } - - // 人脸消失：保留小延迟，避免瞬时抖动导致频繁告别 - faceStableJob?.cancel() - faceStableJob = scope.launch { - delay(AppConfig.Face.STABLE_MS) - if (facePresent) return@launch - if (state != InteractionState.IDLE && state != InteractionState.MEMORY && state != InteractionState.FAREWELL) { - enterFarewell() - } else { - scheduleMemoryMode() + + // 非正脸或人脸消失的处理逻辑 + if (!present) { + faceStableJob?.cancel() + faceStableJob = scope.launch { + delay(AppConfig.Face.STABLE_MS) + if (facePresent) return@launch + if (state != InteractionState.IDLE && state != InteractionState.MEMORY && state != InteractionState.FAREWELL) { + enterFarewell() + } else { + scheduleMemoryMode() + } + } } - } } fun onFaceIdentityUpdated(faceIdentityId: String?, recognized: String?) { @@ -122,6 +148,14 @@ fun onFacePresenceChanged(present: Boolean) { if (!faceIdentityId.isNullOrBlank()) { currentFaceId = faceIdentityId handler.onRememberUser(faceIdentityId, recognized) + // If the pipeline didn't supply a name, look it up from our own store + if (recognized.isNullOrBlank()) { + val storedName = handler.getStoredDisplayName(faceIdentityId) + if (!storedName.isNullOrBlank()) { + recognizedName = storedName + Log.d(TAG, "Loaded stored name for $faceIdentityId: $storedName") + } + } } if (!recognized.isNullOrBlank()) { recognizedName = recognized @@ -170,7 +204,6 @@ fun onFacePresenceChanged(present: Boolean) { return } transitionTo(InteractionState.WAITING_REPLY) - handler.playMotion("haru_g_m17.motion3.json") scheduleWaitingReplyTimeout() } @@ -186,38 +219,30 @@ fun onFacePresenceChanged(present: Boolean) { handler.requestLocalThought(prompt) { greeting -> scope.launch { if (!greeting.isNullOrBlank()) { - // 使用LLM生成的问候语 - handler.playMotion(if (!recognizedName.isNullOrBlank()) "haru_g_m22.motion3.json" else "haru_g_m01.motion3.json") handler.speak(greeting) handler.appendText("\n[问候] $greeting\n") handler.addToChatHistory("assistant", greeting) handler.addAssistantMessageToCloudHistory(greeting) - transitionTo(InteractionState.WAITING_REPLY) - handler.playMotion("haru_g_m17.motion3.json") scheduleWaitingReplyTimeout() } else { - // LLM生成失败，使用默认问候语 useDefaultGreeting() } } } } else { - // 非节日或无网络，使用默认问候语 useDefaultGreeting() } } private fun useDefaultGreeting() { val greeting = smartGreetingUtil.getSmartGreeting(recognizedName) - handler.playMotion(if (!recognizedName.isNullOrBlank()) "haru_g_m22.motion3.json" else "haru_g_m01.motion3.json") handler.speak(greeting) handler.appendText("\n[问候] $greeting\n") handler.addToChatHistory("assistant", greeting) handler.addAssistantMessageToCloudHistory(greeting) transitionTo(InteractionState.WAITING_REPLY) - handler.playMotion("haru_g_m17.motion3.json") scheduleWaitingReplyTimeout() } @@ -247,13 +272,15 @@ fun onFacePresenceChanged(present: Boolean) { // 从数据库获取问题 val topic = getQuestionFromDatabase() - handler.playMotion(if (proactiveRound == 0) "haru_g_m15.motion3.json" else "haru_g_m22.motion3.json") handler.speak("嗨小朋友，可以帮老师个忙吗？"+topic) handler.appendText("\n[主动引导] \"嗨小朋友，可以帮老师个忙吗？\"$topic\n") // 将引导内容添加到对话历史中 handler.addToChatHistory("助手", topic) // 将引导内容添加到云对话历史中 handler.addAssistantMessageToCloudHistory(topic) + + // 触发题目生成检查 + handler.onQuestionAsked(currentFaceId ?: "guest") proactiveJob = scope.launch { hasPendingUserReply = false @@ -268,7 +295,7 @@ fun onFacePresenceChanged(present: Boolean) { askProactiveTopic() } else { transitionTo(InteractionState.WAITING_REPLY) - handler.playMotion("haru_g_m17.motion3.json") +// handler.playMotion("haru_g_m17.motion3.json") scheduleWaitingReplyTimeout() } } @@ -287,26 +314,23 @@ fun onFacePresenceChanged(present: Boolean) { private fun enterFarewell() { transitionTo(InteractionState.FAREWELL) - handler.playMotion("haru_g_idle.motion3.json") +// handler.playMotion("haru_g_idle.motion3.json") handler.speak("再见咯") farewellJob?.cancel() farewellJob = scope.launch { delay(3_000) transitionTo(InteractionState.IDLE) - handler.playMotion("haru_g_idle.motion3.json") scheduleMemoryMode() } } private fun scheduleMemoryMode() { - return // TODO: remove this after testing done! Now just skipping memory memoryJob?.cancel() if (facePresent) return memoryJob = scope.launch { - delay(30_000) + delay(10_000) // wait for profile extraction to finish before using local LLM if (facePresent || state != InteractionState.IDLE) return@launch transitionTo(InteractionState.MEMORY) - handler.playMotion("haru_g_m15.motion3.json") val prompt = "请基于最近互动写一句简短内心想法，口吻温和自然。" handler.requestLocalThought(prompt) { thought -> val finalThought = thought.trim().ifBlank { "我在想，下次见面要聊点有趣的新话题。" } @@ -314,7 +338,6 @@ fun onFacePresenceChanged(present: Boolean) { handler.appendText("\n[回忆] $finalThought\n") if (!facePresent && state == InteractionState.MEMORY) { transitionTo(InteractionState.IDLE) - handler.playMotion("haru_g_idle.motion3.json") scheduleMemoryMode() } } diff --git a/app/src/main/java/com/digitalperson/interaction/UserMemoryStore.kt b/app/src/main/java/com/digitalperson/interaction/UserMemoryStore.kt index 58b842a..0815b12 100644 --- a/app/src/main/java/com/digitalperson/interaction/UserMemoryStore.kt +++ b/app/src/main/java/com/digitalperson/interaction/UserMemoryStore.kt @@ -170,6 +170,14 @@ class UserMemoryStore(context: Context) { fun getRandomUnansweredQuestion(userId: String): Question? { return questionDao.getRandomUnansweredQuestion(userId) } + + fun countUnansweredQuestions(userId: String): Int { + return questionDao.countUnansweredQuestions(userId) + } + + fun getRandomUnansweredQuestions(userId: String, subject: String?, grade: Int?, limit: Int): List { + return questionDao.getRandomUnansweredQuestions(userId, subject, grade, limit) + } fun recordUserAnswer(userId: String, questionId: Long, userAnswer: String?, evaluation: AnswerEvaluation?) { GlobalScope.launch(Dispatchers.IO) { diff --git a/app/src/main/java/com/digitalperson/question/QuestionGenerationAgent.kt b/app/src/main/java/com/digitalperson/question/QuestionGenerationAgent.kt new file mode 100644 index 0000000..e4b64a6 --- /dev/null +++ b/app/src/main/java/com/digitalperson/question/QuestionGenerationAgent.kt @@ -0,0 +1,527 @@ +package com.digitalperson.question + +import android.content.Context +import android.util.Log +import com.digitalperson.config.AppConfig +import com.digitalperson.data.AppDatabase +import com.digitalperson.data.entity.Question +import com.digitalperson.interaction.UserMemoryStore +import com.digitalperson.llm.LLMManager +import kotlinx.coroutines.* +import org.json.JSONArray +import org.json.JSONObject +import java.io.InputStream +import kotlin.coroutines.resume +import kotlin.coroutines.suspendCoroutine + +/** + * 题目生成智能体 + * + * 工作流程： + * 1. 从提示词池中读取一个提示词 + * 2. 调用大模型生成题目 + * 3. 调用大模型审核题目质量 + * 4. 通过审核的题目入库 + * 5. 定期检查题库，题目不足时重新生成 + * + * 特性： + * - 持久化运行 + * - 保证题目多样性 + * - 自动补充题库 + */ +class QuestionGenerationAgent( + private val context: Context, + private val llmManager: LLMManager?, + private val userMemoryStore: UserMemoryStore, + private val config: AgentConfig = AgentConfig(), + private val cloudLLMGenerator: ((String, (String) -> Unit) -> Unit)? = null // 云端LLM生成器 +) { + data class AgentConfig( + val minUnansweredQuestions: Int = 10, // 题库最少题目数量 + val batchSize: Int = 5, // 每次生成的题目数量 + val generationTimeoutMs: Long = 30000, // 生成超时（30秒） + ) + + private val database = AppDatabase.getInstance(context) + private val questionDao = database.questionDao() + private val TAG = "QuestionGenAgent" + + private val agentScope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private var isRunning = false + + // 提示词池 - 可以从文件或资源中加载 + private var promptPool: List = emptyList() + private var currentPromptIndex = 0 + + // 已生成的题目主题记录，用于保证多样性 + private val generatedTopics = mutableSetOf() + + // 最近使用过的提示词索引，避免重复 + private val recentlyUsedPrompts = mutableListOf() + private val maxRecentPrompts = 10 // 最多记录最近使用的10个提示词 + + /** + * 题目提示词 + */ + data class QuestionPrompt( + val subject: String, // 学科：生活语文、生活适应等 + val grade: Int, // 年级 + val topic: String, // 主题 + val difficulty: Int, // 难度 1-5 + val promptTemplate: String, // 提示词模板 + ) + + /** + * 生成的题目数据 + */ + data class GeneratedQuestion( + val content: String, + val answer: String, + val subject: String, + val grade: Int, + val difficulty: Int, + ) + + /** + * 启动智能体 + */ + fun start() { + if (isRunning) { + Log.w(TAG, "Agent is already running") + return + } + + isRunning = true + Log.i(TAG, "Question Generation Agent started") + + // 加载提示词池 + loadPromptPool() + } + + /** + * 停止智能体 + */ + fun stop() { + isRunning = false + agentScope.cancel() + Log.i(TAG, "Question Generation Agent stopped") + } + + /** + * 当一个问题被提问后触发检查 + * 这个方法应该在你的系统每次提问后调用 + */ + fun onQuestionAsked(userId: String) { + if (!isRunning) { + Log.w(TAG, "Agent is not running, ignoring question asked event") + return + } + + Log.d(TAG, "Question asked by user $userId, triggering check...") + + // 异步触发检查和生成 + agentScope.launch { + try { + checkAndGenerateForUser(userId) + } catch (e: Exception) { + Log.e(TAG, "Error checking and generating questions: ${e.message}", e) + } + } + } + + /** + * 检查并为用户生成题目 + */ + private suspend fun checkAndGenerateForUser(userId: String) { + val unansweredCount = userMemoryStore.countUnansweredQuestions(userId) + + if (unansweredCount < config.minUnansweredQuestions) { + // 如果是第一次（题目很少），生成更多题目 + val isInitialLoad = unansweredCount == 0 + val targetCount = if (isInitialLoad) { + config.minUnansweredQuestions * 2 // 首次加载生成双倍题目 + } else { + config.minUnansweredQuestions + } + + val neededCount = targetCount - unansweredCount + + Log.i(TAG, "User $userId has $unansweredCount unanswered questions (initial=$isInitialLoad), generating $neededCount more...") + + // 获取用户信息，用于个性化生成 + val userProfile = userMemoryStore.getMemory(userId) + + // 批量生成题目 + repeat(neededCount) { index -> + if (!isRunning) return@repeat + + try { + Log.d(TAG, "Generating question ${index + 1}/$neededCount for user $userId") + generateOneQuestion(userId, userProfile) + delay(1000) // 避免频繁调用 + } catch (e: Exception) { + Log.e(TAG, "Failed to generate question ${index + 1}: ${e.message}", e) + } + } + + Log.i(TAG, "Finished generating questions for user $userId") + } else { + Log.d(TAG, "User $userId has $unansweredCount unanswered questions, no need to generate") + } + } + + /** + * 生成单个题目 + */ + private fun generateOneQuestion(userId: String, userProfile: com.digitalperson.data.entity.UserMemory?) { + // 1. 获取下一个提示词 + val prompt = getNextPrompt() + if (prompt == null) { + Log.w(TAG, "No prompt available") + return + } + + // 2. 构建生成提示词 + val generationPrompt = buildGenerationPrompt(prompt, userProfile) + + // 3. 调用大模型生成题目 + generateQuestionFromLLM(generationPrompt) { generatedQuestion -> + if (generatedQuestion == null) { + Log.w(TAG, "Failed to generate question") + return@generateQuestionFromLLM + } + + // 4. 审核题目 + val reviewPrompt = buildReviewPrompt(generatedQuestion) + reviewQuestionWithLLM(reviewPrompt) { reviewPassed -> + if (!reviewPassed) { + Log.w(TAG, "Question review failed") + return@reviewQuestionWithLLM + } + + // 5. 入库 + val question = Question( + id = 0, + content = generatedQuestion.content, + answer = generatedQuestion.answer, + subject = generatedQuestion.subject, + grade = generatedQuestion.grade, + difficulty = generatedQuestion.difficulty, + createdAt = System.currentTimeMillis() + ) + + questionDao.insert(question) + Log.i(TAG, "Question saved: ${question.content.take(30)}...") + + // 记录已生成的主题 + generatedTopics.add("${question.subject}_${question.grade}_${question.difficulty}") + } + } + } + + /** + * 构建题目生成提示词 + */ + private fun buildGenerationPrompt( + prompt: QuestionPrompt, + userProfile: com.digitalperson.data.entity.UserMemory? + ): String { + val userInfo = buildString { + if (userProfile != null) { + append("用户信息：") + userProfile.displayName?.let { append("姓名：$it，") } + userProfile.age?.let { append("年龄：$it，") } + userProfile.gender?.let { append("性别：$it，") } + userProfile.hobbies?.let { append("爱好：$it，") } + append("\n") + } + } + + return """ + 你是一个专门为特殊教育儿童设计题目的教育专家。请根据以下要求生成一个题目： + + $userInfo + 学科：${prompt.subject} + 年级：${prompt.grade} + 主题：${prompt.topic} + 难度：${prompt.difficulty}（1-5，5最难） + + 具体要求： + ${prompt.promptTemplate} + + 通用要求： + 1. 题目要贴近生活，适合智力障碍儿童理解 + 2. 语言简单明了，避免复杂句式 + 3. 题目内容积极向上 + 4. 提供标准答案 + 5. 确保题目没有重复（避免常见的基础题目如"1+1=?"） + 6. 题目要有趣味性，能吸引学生注意力 + + 请以JSON格式返回，格式如下： + { + "content": "题目内容", + "answer": "标准答案", + "explanation": "题目解析（可选）" + } + + 只返回JSON，不要其他内容。 + """.trimIndent() + } + + /** + * 构建题目审核提示词 + */ + private fun buildReviewPrompt(question: GeneratedQuestion): String { + return """ + 你是一个教育题目审核专家。请审核以下题目是否适合特殊教育儿童： + + 学科：${question.subject} + 年级：${question.grade} + 难度：${question.difficulty} + 题目：${question.content} + 答案：${question.answer} + + 审核标准： + 1. 题目是否清晰明确？ + 2. 难度是否适合对应年级的特殊教育儿童？ + 3. 内容是否积极向上？ + 4. 语言是否简单易懂？ + 5. 答案是否正确？ + + 请以JSON格式返回审核结果： + { + "passed": true/false, + "reason": "审核意见" + } + + 只返回JSON，不要其他内容。 + """.trimIndent() + } + + /** + * 调用LLM生成题目 + */ + private fun generateQuestionFromLLM(prompt: String, onResult: (GeneratedQuestion?) -> Unit) { + // 优先使用本地LLM，如果不可用则使用云端LLM + if (llmManager != null) { + // 使用本地LLM + llmManager.generate(prompt) { response: String -> + parseGeneratedQuestion(response, onResult) + } + } else if (cloudLLMGenerator != null) { + // 使用云端LLM + Log.d(TAG, "Using cloud LLM to generate question") + cloudLLMGenerator.invoke(prompt) { response -> + parseGeneratedQuestion(response, onResult) + } + } else { + Log.e(TAG, "No LLM available (neither local nor cloud)") + onResult(null) + } + } + + /** + * 解析生成的题目 + */ + private fun parseGeneratedQuestion(response: String, onResult: (GeneratedQuestion?) -> Unit) { + try { + val json = extractJsonFromResponse(response) + if (json != null) { + val question = GeneratedQuestion( + content = json.getString("content"), + answer = json.getString("answer"), + subject = "生活适应", + grade = 1, + difficulty = 1 + ) + onResult(question) + } else { + onResult(null) + } + } catch (e: Exception) { + Log.e(TAG, "Failed to parse generated question: ${e.message}") + onResult(null) + } + } + + /** + * 调用LLM审核题目 + */ + private fun reviewQuestionWithLLM(prompt: String, onResult: (Boolean) -> Unit) { + // 优先使用本地LLM，如果不可用则使用云端LLM + if (llmManager != null) { + // 使用本地LLM + llmManager.generate(prompt) { response: String -> + parseReviewResult(response, onResult) + } + } else if (cloudLLMGenerator != null) { + // 使用云端LLM + Log.d(TAG, "Using cloud LLM to review question") + cloudLLMGenerator.invoke(prompt) { response -> + parseReviewResult(response, onResult) + } + } else { + Log.e(TAG, "No LLM available for review (neither local nor cloud)") + onResult(false) + } + } + + /** + * 解析审核结果 + */ + private fun parseReviewResult(response: String, onResult: (Boolean) -> Unit) { + try { + val json = extractJsonFromResponse(response) + if (json != null) { + val passed = json.getBoolean("passed") + onResult(passed) + } else { + onResult(false) + } + } catch (e: Exception) { + Log.e(TAG, "Failed to parse review result: ${e.message}") + onResult(false) + } + } + + /** + * 从响应中提取JSON + */ + private fun extractJsonFromResponse(response: String): JSONObject? { + val trimmed = response.trim() + val start = trimmed.indexOf('{') + val end = trimmed.lastIndexOf('}') + + if (start >= 0 && end > start) { + val jsonStr = trimmed.substring(start, end + 1) + return JSONObject(jsonStr) + } + return null + } + + /** + * 获取下一个提示词（智能选择，避免重复） + */ + private fun getNextPrompt(): QuestionPrompt? { + if (promptPool.isEmpty()) return null + + // 选择一个最近未使用过的提示词 + var promptIndex = -1 + + // 尝试找到一个未在最近使用列表中出现的提示词 + for (i in 0 until promptPool.size) { + val candidateIndex = currentPromptIndex % promptPool.size + currentPromptIndex++ + + if (!recentlyUsedPrompts.contains(candidateIndex)) { + promptIndex = candidateIndex + break + } + } + + // 如果所有提示词都在最近使用列表中，选择最早使用的那个 + if (promptIndex == -1) { + promptIndex = currentPromptIndex % promptPool.size + currentPromptIndex++ + } + + // 记录到最近使用列表 + recentlyUsedPrompts.add(promptIndex) + if (recentlyUsedPrompts.size > maxRecentPrompts) { + recentlyUsedPrompts.removeAt(0) + } + + return promptPool[promptIndex] + } + + /** + * 加载提示词池 + */ + private fun loadPromptPool() { + try { + val inputStream: InputStream = context.assets.open("question_prompts.json") + val jsonString = inputStream.bufferedReader().use { it.readText() } + val jsonArray = JSONArray(jsonString) + + val prompts = mutableListOf() + for (i in 0 until jsonArray.length()) { + val json = jsonArray.getJSONObject(i) + val prompt = QuestionPrompt( + subject = json.getString("subject"), + grade = json.getInt("grade"), + topic = json.getString("topic"), + difficulty = json.getInt("difficulty"), + promptTemplate = json.getString("promptTemplate") + ) + prompts.add(prompt) + } + + promptPool = prompts + Log.i(TAG, "Loaded ${promptPool.size} prompts from JSON file") + } catch (e: Exception) { + Log.e(TAG, "Failed to load prompts from JSON: ${e.message}") + // Fallback to default prompts if JSON loading fails + promptPool = getDefaultPrompts() + Log.i(TAG, "Loaded ${promptPool.size} default prompts") + } + } + + /** + * 获取默认提示词（备用） + */ + private fun getDefaultPrompts(): List { + return listOf( + QuestionPrompt( + subject = "生活适应", + grade = 1, + topic = "认识家庭成员", + difficulty = 1, + promptTemplate = "生成一个关于认识家庭成员的题目" + ), + QuestionPrompt( + subject = "生活适应", + grade = 1, + topic = "认识日常用品", + difficulty = 1, + promptTemplate = "生成一个关于认识日常用品的题目" + ), + QuestionPrompt( + subject = "生活适应", + grade = 1, + topic = "认识天气", + difficulty = 1, + promptTemplate = "生成一个关于认识天气的题目" + ), + QuestionPrompt( + subject = "生活语文", + grade = 1, + topic = "认识汉字", + difficulty = 1, + promptTemplate = "生成一个关于认识简单汉字的题目" + ), + QuestionPrompt( + subject = "生活语文", + grade = 1, + topic = "简单句子理解", + difficulty = 2, + promptTemplate = "生成一个关于简单句子理解的题目" + ), + ) + } + + /** + * 获取所有已知用户 + */ + private fun getAllKnownUsers(): List { + // 从 UserMemoryStore 获取所有用户 + // 这里需要根据实际实现调整 + return listOf("guest") // 临时返回，实际应该从数据库获取 + } + + /** + * 手动触发一次生成（用于测试） + */ + suspend fun triggerGeneration(userId: String) { + checkAndGenerateForUser(userId) + } +} diff --git a/app/src/main/java/com/digitalperson/tts/QCloudTtsManager.kt b/app/src/main/java/com/digitalperson/tts/QCloudTtsManager.kt index 03a904e..4f8d136 100644 --- a/app/src/main/java/com/digitalperson/tts/QCloudTtsManager.kt +++ b/app/src/main/java/com/digitalperson/tts/QCloudTtsManager.kt @@ -108,7 +108,8 @@ class QCloudTtsManager(private val context: Context) { if (ttsStopped.get()) { ttsStopped.set(false) } - val cleanedSeg = seg.trimEnd('.', '。', '!', '！', '?', '？', ',', '，', ';', '；', ':', '：') + val cleanedSeg = seg.replace(Regex("\\[.*?\\]"), "") + .trimEnd('.', '。', '!', '！', '?', '？', ',', '，', ';', '；', ':', '：') if (cleanedSeg.isBlank()) return val now = System.currentTimeMillis() // Guard against accidental duplicate enqueue from streaming/final flush overlap. diff --git a/app/src/main/java/com/digitalperson/tts/TtsController.kt b/app/src/main/java/com/digitalperson/tts/TtsController.kt index 7a6d5e7..4696cd2 100644 --- a/app/src/main/java/com/digitalperson/tts/TtsController.kt +++ b/app/src/main/java/com/digitalperson/tts/TtsController.kt @@ -2,6 +2,7 @@ package com.digitalperson.tts import android.content.Context import android.util.Log +import com.digitalperson.mood.MoodManager class TtsController(private val context: Context) { @@ -144,13 +145,38 @@ class TtsController(private val context: Context) { } fun enqueueSegment(seg: String) { + val cleaned = seg.replace(Regex("\\[.*?\\]"), "").trim() + if (cleaned.isEmpty()) return if (useQCloudTts) { - qcloudTts?.enqueueSegment(seg) + qcloudTts?.enqueueSegment(cleaned) } else { - localTts?.enqueueSegment(seg) + localTts?.enqueueSegment(cleaned) } } + /** + * Extracts the mood tag from an LLM response, updates [MoodManager], notifies the caller + * via [onMoodChanged] if the mood changed, and enqueues the cleaned text for TTS playback. + * + * Use this instead of manually calling extractAndFilterMood + enqueueSegment + enqueueEnd. + * + * @param response Raw LLM response text (may contain a leading [mood] tag). + * @param onMoodChanged Called on the calling thread when the mood has changed; receives the new mood string. + * @return The filtered text that was enqueued, or null if the text was empty after cleaning. + */ + fun speakLlmResponse(response: String, onMoodChanged: ((mood: String) -> Unit)? = null): String? { + val previousMood = MoodManager.getCurrentMood() + val (filteredText, mood) = MoodManager.extractAndFilterMood(response.trim()) + Log.d(TAG, "speakLlmResponse mood=$mood text=${filteredText.take(60)}") + if (mood != previousMood) { + onMoodChanged?.invoke(mood) + } + if (filteredText.isBlank()) return null + enqueueSegment(filteredText) + enqueueEnd() + return filteredText + } + fun enqueueEnd() { if (useQCloudTts) { qcloudTts?.enqueueEnd() diff --git a/app/src/main/java/com/digitalperson/tts/TtsManager.kt b/app/src/main/java/com/digitalperson/tts/TtsManager.kt index 4366087..5b49781 100644 --- a/app/src/main/java/com/digitalperson/tts/TtsManager.kt +++ b/app/src/main/java/com/digitalperson/tts/TtsManager.kt @@ -10,6 +10,7 @@ import android.widget.Toast import com.digitalperson.config.AppConfig import com.digitalperson.metrics.TraceManager import com.digitalperson.metrics.TraceSession +import com.digitalperson.mood.MoodManager import com.k2fsa.sherpa.onnx.OfflineTts import com.k2fsa.sherpa.onnx.getOfflineTtsConfig import kotlinx.coroutines.CoroutineScope @@ -131,7 +132,9 @@ class TtsManager(private val context: Context) { // Recover from interrupt state for next turn. ttsStopped.set(false) } - val cleanedSeg = seg.trimEnd('.', '。', '!', '！', '?', '？', ',', '，', ';', '；', ':', '：') + val cleanedSeg = seg.replace(Regex("\\[.*?\\]"), "") + .trimEnd('.', '。', '!', '！', '?', '？', ',', '，', ';', '；', ':', '：') + if (cleanedSeg.isBlank()) return callback?.onTraceMarkTtsRequestEnqueued() ttsQueue.offer(TtsQueueItem.Segment(cleanedSeg)) @@ -141,6 +144,25 @@ class TtsManager(private val context: Context) { fun enqueueEnd() { ttsQueue.offer(TtsQueueItem.End) } + + /** + * Convenience wrapper: extracts mood tag from an LLM response, updates [MoodManager], + * notifies [onMoodChanged] if the mood changed, then enqueues the cleaned text + End marker. + * + * @return The filtered text enqueued, or null if blank after cleaning. + */ + fun speakLlmResponse(response: String, onMoodChanged: ((mood: String) -> Unit)? = null): String? { + val previousMood = MoodManager.getCurrentMood() + val (filteredText, mood) = MoodManager.extractAndFilterMood(response.trim()) + Log.d(TAG, "speakLlmResponse mood=$mood text=${filteredText.take(60)}") + if (mood != previousMood) { + onMoodChanged?.invoke(mood) + } + if (filteredText.isBlank()) return null + enqueueSegment(filteredText) + enqueueEnd() + return filteredText + } fun isPlaying(): Boolean = ttsPlaying.get() diff --git a/app/src/main/java/com/digitalperson/ui/Live2DUiManager.kt b/app/src/main/java/com/digitalperson/ui/Live2DUiManager.kt index cef9e1b..779e385 100644 --- a/app/src/main/java/com/digitalperson/ui/Live2DUiManager.kt +++ b/app/src/main/java/com/digitalperson/ui/Live2DUiManager.kt @@ -1,16 +1,26 @@ package com.digitalperson.ui +import android.animation.ObjectAnimator import android.app.Activity import android.app.ProgressDialog +import android.content.Context +import android.content.res.ColorStateList +import android.graphics.Color import android.opengl.GLSurfaceView +import android.os.Build +import android.os.VibrationEffect +import android.os.Vibrator import android.text.method.ScrollingMovementMethod import android.view.MotionEvent +import android.view.View import android.widget.Button +import android.widget.FrameLayout import android.widget.LinearLayout import android.widget.ScrollView import android.widget.Switch import android.widget.TextView import android.widget.Toast +import com.digitalperson.R import com.digitalperson.live2d.Live2DAvatarManager class Live2DUiManager(private val activity: Activity) { @@ -19,11 +29,14 @@ class Live2DUiManager(private val activity: Activity) { private var startButton: Button? = null private var stopButton: Button? = null private var recordButton: Button? = null + private var recordButtonContainer: FrameLayout? = null + private var recordButtonGlow: View? = null private var traditionalButtons: LinearLayout? = null private var llmModeSwitch: Switch? = null private var llmModeSwitchRow: LinearLayout? = null private var avatarManager: Live2DAvatarManager? = null private var downloadProgressDialog: ProgressDialog? = null + private var pulseAnimator: ObjectAnimator? = null private var lastUiText: String = "" @@ -44,7 +57,11 @@ class Live2DUiManager(private val activity: Activity) { scrollView = activity.findViewById(scrollViewId) if (startButtonId != -1) startButton = activity.findViewById(startButtonId) if (stopButtonId != -1) stopButton = activity.findViewById(stopButtonId) - if (recordButtonId != -1) recordButton = activity.findViewById(recordButtonId) + if (recordButtonId != -1) { + recordButton = activity.findViewById(recordButtonId) + recordButtonContainer = activity.findViewById(R.id.record_button_container) + recordButtonGlow = activity.findViewById(R.id.record_button_glow) + } if (traditionalButtonsId != -1) traditionalButtons = activity.findViewById(traditionalButtonsId) if (llmModeSwitchId != -1) llmModeSwitch = activity.findViewById(llmModeSwitchId) if (llmModeSwitchRowId != -1) llmModeSwitchRow = activity.findViewById(llmModeSwitchRowId) @@ -59,6 +76,41 @@ class Live2DUiManager(private val activity: Activity) { llmModeSwitchRow?.visibility = LinearLayout.GONE } + private fun vibrate(isPress: Boolean) { + val vibrator = activity.getSystemService(Context.VIBRATOR_SERVICE) as? Vibrator + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val effect = if (isPress) + VibrationEffect.createPredefined(VibrationEffect.EFFECT_CLICK) + else + VibrationEffect.createPredefined(VibrationEffect.EFFECT_TICK) + vibrator?.vibrate(effect) + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val duration = if (isPress) 30L else 15L + vibrator?.vibrate(VibrationEffect.createOneShot(duration, VibrationEffect.DEFAULT_AMPLITUDE)) + } else { + @Suppress("DEPRECATION") + vibrator?.vibrate(if (isPress) 30L else 15L) + } + } + + private fun startPulseAnimation() { + val glow = recordButtonGlow ?: return + glow.visibility = View.VISIBLE + glow.alpha = 0.6f + pulseAnimator = ObjectAnimator.ofFloat(glow, "alpha", 0.6f, 0.2f).apply { + duration = 800 + repeatMode = ObjectAnimator.REVERSE + repeatCount = ObjectAnimator.INFINITE + start() + } + } + + private fun stopPulseAnimation() { + pulseAnimator?.cancel() + pulseAnimator = null + recordButtonGlow?.visibility = View.GONE + } + fun setStartButtonListener(listener: () -> Unit) { startButton?.setOnClickListener { listener() } } @@ -73,12 +125,22 @@ class Live2DUiManager(private val activity: Activity) { when (event.action) { MotionEvent.ACTION_DOWN -> { recordButton?.isPressed = true + vibrate(true) + recordButton?.text = "松开结束" + recordButton?.backgroundTintList = ColorStateList.valueOf(Color.parseColor("#F44336")) + recordButton?.animate()?.scaleX(1.15f)?.scaleY(1.15f)?.setDuration(100)?.start() + startPulseAnimation() listener(true) true } MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> { recordButton?.isPressed = false + vibrate(false) + recordButton?.text = "按住说话" + recordButton?.backgroundTintList = ColorStateList.valueOf(Color.parseColor("#4CAF50")) + recordButton?.animate()?.scaleX(1.0f)?.scaleY(1.0f)?.setDuration(100)?.start() + stopPulseAnimation() listener(false) true } @@ -90,10 +152,10 @@ class Live2DUiManager(private val activity: Activity) { fun setUseHoldToSpeak(useHoldToSpeak: Boolean) { if (useHoldToSpeak) { traditionalButtons?.visibility = LinearLayout.GONE - recordButton?.visibility = Button.VISIBLE + recordButtonContainer?.visibility = FrameLayout.VISIBLE } else { traditionalButtons?.visibility = LinearLayout.VISIBLE - recordButton?.visibility = Button.GONE + recordButtonContainer?.visibility = FrameLayout.GONE } } diff --git a/app/src/main/java/com/digitalperson/ui/UiManager.kt b/app/src/main/java/com/digitalperson/ui/UiManager.kt index 8d7dd5c..d2aefb8 100644 --- a/app/src/main/java/com/digitalperson/ui/UiManager.kt +++ b/app/src/main/java/com/digitalperson/ui/UiManager.kt @@ -1,15 +1,25 @@ package com.digitalperson.ui +import android.animation.ObjectAnimator import android.app.Activity +import android.content.Context +import android.content.res.ColorStateList +import android.graphics.Color +import android.os.Build +import android.os.VibrationEffect +import android.os.Vibrator import android.text.method.ScrollingMovementMethod import android.util.Log import android.view.MotionEvent +import android.view.View import android.widget.Button +import android.widget.FrameLayout import android.widget.ScrollView import android.widget.TextView import android.widget.Toast import com.digitalperson.config.AppConfig import com.digitalperson.player.VideoPlayerManager +import com.digitalperson.R import com.google.android.exoplayer2.ui.PlayerView class UiManager(private val activity: Activity) { @@ -19,7 +29,10 @@ class UiManager(private val activity: Activity) { private var startButton: Button? = null private var stopButton: Button? = null private var recordButton: Button? = null + private var recordButtonContainer: FrameLayout? = null + private var recordButtonGlow: View? = null private var videoPlayerManager: VideoPlayerManager? = null + private var pulseAnimator: ObjectAnimator? = null private var lastUiText: String = "" @@ -36,7 +49,11 @@ class UiManager(private val activity: Activity) { scrollView = activity.findViewById(scrollViewId) if (startButtonId != -1) startButton = activity.findViewById(startButtonId) if (stopButtonId != -1) stopButton = activity.findViewById(stopButtonId) - if (recordButtonId != -1) recordButton = activity.findViewById(recordButtonId) + if (recordButtonId != -1) { + recordButton = activity.findViewById(recordButtonId) + recordButtonContainer = activity.findViewById(R.id.record_button_container) + recordButtonGlow = activity.findViewById(R.id.record_button_glow) + } textView?.movementMethod = ScrollingMovementMethod() @@ -49,6 +66,41 @@ class UiManager(private val activity: Activity) { Log.w(AppConfig.TAG, "PlayerViews not found or init failed: ${e.message}") } } + + private fun vibrate(isPress: Boolean) { + val vibrator = activity.getSystemService(Context.VIBRATOR_SERVICE) as? Vibrator + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val effect = if (isPress) + VibrationEffect.createPredefined(VibrationEffect.EFFECT_CLICK) + else + VibrationEffect.createPredefined(VibrationEffect.EFFECT_TICK) + vibrator?.vibrate(effect) + } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val duration = if (isPress) 30L else 15L + vibrator?.vibrate(VibrationEffect.createOneShot(duration, VibrationEffect.DEFAULT_AMPLITUDE)) + } else { + @Suppress("DEPRECATION") + vibrator?.vibrate(if (isPress) 30L else 15L) + } + } + + private fun startPulseAnimation() { + val glow = recordButtonGlow ?: return + glow.visibility = View.VISIBLE + glow.alpha = 0.6f + pulseAnimator = ObjectAnimator.ofFloat(glow, "alpha", 0.6f, 0.2f).apply { + duration = 800 + repeatMode = ObjectAnimator.REVERSE + repeatCount = ObjectAnimator.INFINITE + start() + } + } + + private fun stopPulseAnimation() { + pulseAnimator?.cancel() + pulseAnimator = null + recordButtonGlow?.visibility = View.GONE + } fun setStartButtonListener(listener: () -> Unit) { startButton?.setOnClickListener { listener() } @@ -63,11 +115,21 @@ class UiManager(private val activity: Activity) { _, event -> when (event.action) { MotionEvent.ACTION_DOWN -> { + vibrate(true) + recordButton?.text = "松开结束" + recordButton?.backgroundTintList = ColorStateList.valueOf(Color.parseColor("#F44336")) + recordButton?.animate()?.scaleX(1.15f)?.scaleY(1.15f)?.setDuration(100)?.start() + startPulseAnimation() listener(true) true } MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> { + vibrate(false) + recordButton?.text = "按住说话" + recordButton?.backgroundTintList = ColorStateList.valueOf(Color.parseColor("#4CAF50")) + recordButton?.animate()?.scaleX(1.0f)?.scaleY(1.0f)?.setDuration(100)?.start() + stopPulseAnimation() listener(false) true } diff --git a/app/src/main/res/drawable/record_button_glow.xml b/app/src/main/res/drawable/record_button_glow.xml new file mode 100644 index 0000000..255a29a --- /dev/null +++ b/app/src/main/res/drawable/record_button_glow.xml @@ -0,0 +1,3 @@ + + + diff --git a/app/src/main/res/layout/activity_live2d_chat.xml b/app/src/main/res/layout/activity_live2d_chat.xml index f4bbe98..3e16385 100644 --- a/app/src/main/res/layout/activity_live2d_chat.xml +++ b/app/src/main/res/layout/activity_live2d_chat.xml @@ -170,21 +170,36 @@ -