llm testing
This commit is contained in:
gcw_4spBpAfv
@@ -42,6 +42,7 @@ import com.digitalperson.data.AppDatabase
|
||||
import com.digitalperson.data.entity.ChatMessage
|
||||
import com.digitalperson.interaction.ConversationBufferMemory
|
||||
import com.digitalperson.interaction.ConversationSummaryMemory
|
||||
|
||||
import java.io.File
|
||||
import android.graphics.BitmapFactory
|
||||
import org.json.JSONObject
|
||||
@@ -56,6 +57,9 @@ import kotlinx.coroutines.isActive
|
||||
import kotlinx.coroutines.launch
|
||||
import kotlinx.coroutines.withContext
|
||||
|
||||
import com.digitalperson.onboard_testing.FaceRecognitionTest
|
||||
import com.digitalperson.onboard_testing.LLMSummaryTest
|
||||
|
||||
class Live2DChatActivity : AppCompatActivity() {
|
||||
companion object {
|
||||
private const val TAG_ACTIVITY = "Live2DChatActivity"
|
||||
@@ -116,6 +120,9 @@ class Live2DChatActivity : AppCompatActivity() {
|
||||
private var lastFaceIdentityId: String? = null
|
||||
private var lastFaceRecognizedName: String? = null
|
||||
|
||||
private lateinit var faceRecognitionTest: FaceRecognitionTest
|
||||
private lateinit var llmSummaryTest: LLMSummaryTest
|
||||
|
||||
override fun onRequestPermissionsResult(
|
||||
requestCode: Int,
|
||||
permissions: Array<String>,
|
||||
@@ -400,6 +407,8 @@ class Live2DChatActivity : AppCompatActivity() {
|
||||
// 显示本地 LLM 开关，并同步状态
|
||||
uiManager.showLLMSwitch(false)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -457,12 +466,21 @@ class Live2DChatActivity : AppCompatActivity() {
|
||||
}
|
||||
|
||||
// 测试人脸识别（延迟执行，确保所有组件初始化完成）
|
||||
// ioScope.launch {
|
||||
// kotlinx.coroutines.delay(10000) // 等待3秒，确保所有组件初始化完成
|
||||
// runOnUiThread {
|
||||
// runFaceRecognitionTest()
|
||||
// }
|
||||
// }
|
||||
if (AppConfig.OnboardTesting.FACE_REGONITION) {
|
||||
faceRecognitionTest = FaceRecognitionTest(this)
|
||||
faceRecognitionTest.setFaceDetectionPipeline(faceDetectionPipeline)
|
||||
CoroutineScope(Dispatchers.IO).launch {
|
||||
kotlinx.coroutines.delay(10000)
|
||||
runOnUiThread {
|
||||
faceRecognitionTest.runTest("http://192.168.1.19:5000/api/face_test_images") { message ->
|
||||
Log.i(AppConfig.TAG, message)
|
||||
uiManager.appendToUi("\n$message\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -479,185 +497,8 @@ class Live2DChatActivity : AppCompatActivity() {
|
||||
.show()
|
||||
}
|
||||
|
||||
/**
|
||||
* 运行人脸识别相似度测试
|
||||
* 使用网络服务器上的测试图片
|
||||
*/
|
||||
private fun runFaceRecognitionTest() {
|
||||
Log.i(TAG_ACTIVITY, "Starting face recognition test...")
|
||||
uiManager.appendToUi("\n[测试] 开始人脸识别相似度测试...\n")
|
||||
|
||||
// 从服务器获取目录下的所有图片文件列表
|
||||
ioScope.launch {
|
||||
try {
|
||||
val imageUrls = fetchImageListFromServer("http://192.168.1.19:5000/api/face_test_images")
|
||||
|
||||
if (imageUrls.isEmpty()) {
|
||||
Log.e(AppConfig.TAG, "No images found in server directory")
|
||||
runOnUiThread {
|
||||
uiManager.appendToUi("\n[测试] 服务器目录中没有找到图片文件\n")
|
||||
}
|
||||
return@launch
|
||||
}
|
||||
|
||||
Log.i(AppConfig.TAG, "[测试]Found ${imageUrls.size} images: $imageUrls")
|
||||
runOnUiThread {
|
||||
uiManager.appendToUi("\n[测试] 发现 ${imageUrls.size} 张测试图片\n")
|
||||
}
|
||||
|
||||
val bitmaps = mutableListOf<Pair<String, Bitmap>>()
|
||||
|
||||
// 下载所有图片
|
||||
for (url in imageUrls) {
|
||||
Log.d(AppConfig.TAG, "[测试]Downloading test image: $url")
|
||||
val bitmap = downloadImage(url)
|
||||
if (bitmap != null) {
|
||||
val fileName = url.substringAfterLast("/")
|
||||
bitmaps.add(fileName to bitmap)
|
||||
Log.d(AppConfig.TAG, "[测试]Downloaded image $fileName successfully")
|
||||
} else {
|
||||
Log.e(AppConfig.TAG, "[测试]Failed to download image: $url")
|
||||
}
|
||||
}
|
||||
|
||||
if (bitmaps.size < 2) {
|
||||
Log.e(AppConfig.TAG, "[测试]Not enough test images downloaded")
|
||||
runOnUiThread {
|
||||
uiManager.appendToUi("\n[测试] 测试图片下载失败，无法进行测试\n")
|
||||
}
|
||||
return@launch
|
||||
}
|
||||
|
||||
// 对所有图片两两比较
|
||||
Log.i(AppConfig.TAG, "[测试]Starting similarity comparison for ${bitmaps.size} images...")
|
||||
for (i in 0 until bitmaps.size) {
|
||||
for (j in i + 1 until bitmaps.size) {
|
||||
val (fileName1, bitmap1) = bitmaps[i]
|
||||
val (fileName2, bitmap2) = bitmaps[j]
|
||||
|
||||
Log.d(AppConfig.TAG, "[测试]Comparing $fileName1 with $fileName2")
|
||||
|
||||
// 检测人脸
|
||||
val face1 = detectFace(bitmap1)
|
||||
val face2 = detectFace(bitmap2)
|
||||
|
||||
Log.d(AppConfig.TAG, "[测试]Face detection result: face1=$face1, face2=$face2")
|
||||
|
||||
if (face1 != null && face2 != null) {
|
||||
// 计算相似度
|
||||
Log.d(AppConfig.TAG, "[测试]Detected faces, calculating similarity...")
|
||||
val similarity = faceDetectionPipeline?.getRecognizer()?.testSimilarityBetween(
|
||||
bitmap1, face1, bitmap2, face2
|
||||
)
|
||||
val similarityRaw = faceDetectionPipeline?.getRecognizer()?.run {
|
||||
val emb1 = extractEmbedding(bitmap1, face1)
|
||||
val emb2 = extractEmbedding(bitmap2, face2)
|
||||
if (emb1.isNotEmpty() && emb2.isNotEmpty()) {
|
||||
var dot = 0f
|
||||
var n1 = 0f
|
||||
var n2 = 0f
|
||||
for (k in emb1.indices) {
|
||||
dot += emb1[k] * emb2[k]
|
||||
n1 += emb1[k] * emb1[k]
|
||||
n2 += emb2[k] * emb2[k]
|
||||
}
|
||||
if (n1 > 1e-12f && n2 > 1e-12f) {
|
||||
(dot / (kotlin.math.sqrt(n1) * kotlin.math.sqrt(n2))).coerceIn(-1f, 1f)
|
||||
} else -1f
|
||||
} else -1f
|
||||
}
|
||||
|
||||
Log.d(AppConfig.TAG, "[测试]Similarity result: $similarity")
|
||||
|
||||
if (similarity != null && similarity >= 0) {
|
||||
val message = "[测试] 图片 $fileName1 与 $fileName2 的相似度: $similarity"
|
||||
val compareMessage = "[测试] 对齐后=$similarity, 原始裁剪=$similarityRaw"
|
||||
Log.i(AppConfig.TAG, message)
|
||||
Log.i(AppConfig.TAG, compareMessage)
|
||||
runOnUiThread {
|
||||
uiManager.appendToUi("\n$message\n")
|
||||
uiManager.appendToUi("$compareMessage\n")
|
||||
}
|
||||
} else {
|
||||
Log.w(AppConfig.TAG, "[测试]Failed to calculate similarity: $similarity")
|
||||
runOnUiThread {
|
||||
uiManager.appendToUi("\n[测试] 计算相似度失败: $similarity\n")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
val message = "[测试] 无法检测到人脸: $fileName1 或 $fileName2"
|
||||
Log.w(AppConfig.TAG, message)
|
||||
runOnUiThread {
|
||||
uiManager.appendToUi("\n$message\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Log.i(AppConfig.TAG, "[测试]Face recognition test completed")
|
||||
runOnUiThread {
|
||||
uiManager.appendToUi("\n[测试] 人脸识别相似度测试完成\n")
|
||||
}
|
||||
|
||||
} catch (e: Exception) {
|
||||
Log.e(AppConfig.TAG, "Error during face recognition test: ${e.message}", e)
|
||||
runOnUiThread {
|
||||
uiManager.appendToUi("\n[测试] 测试过程中发生错误: ${e.message}\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从服务器获取目录下的图片文件列表
|
||||
* 调用 API 接口获取图片列表
|
||||
*/
|
||||
private fun fetchImageListFromServer(apiUrl: String): List<String> {
|
||||
val imageUrls = mutableListOf<String>()
|
||||
|
||||
return try {
|
||||
// 调用 API 接口
|
||||
val connection = java.net.URL(apiUrl).openConnection() as java.net.HttpURLConnection
|
||||
connection.requestMethod = "GET"
|
||||
connection.connectTimeout = 10000
|
||||
connection.readTimeout = 10000
|
||||
connection.setRequestProperty("Accept", "application/json")
|
||||
|
||||
try {
|
||||
val responseCode = connection.responseCode
|
||||
if (responseCode == 200) {
|
||||
connection.inputStream.use { input ->
|
||||
val content = input.bufferedReader().use { it.readText() }
|
||||
Log.d(AppConfig.TAG, "API response: $content")
|
||||
|
||||
// 解析 JSON 响应
|
||||
val jsonObject = org.json.JSONObject(content)
|
||||
val imagesArray = jsonObject.getJSONArray("images")
|
||||
|
||||
// 构建完整的图片 URL
|
||||
val baseUrl = apiUrl.replace("/api/face_test_images", "/shared_files/face_test")
|
||||
for (i in 0 until imagesArray.length()) {
|
||||
val fileName = imagesArray.getString(i)
|
||||
val fullUrl = "$baseUrl/$fileName"
|
||||
imageUrls.add(fullUrl)
|
||||
Log.d(AppConfig.TAG, "Added image URL: $fullUrl")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Log.e(AppConfig.TAG, "API request failed with code: $responseCode")
|
||||
}
|
||||
} finally {
|
||||
connection.disconnect()
|
||||
}
|
||||
|
||||
imageUrls
|
||||
} catch (e: Exception) {
|
||||
Log.e(AppConfig.TAG, "Failed to fetch image list: ${e.message}", e)
|
||||
// 如果获取失败，返回空列表
|
||||
emptyList()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 检查 URL 是否存在
|
||||
*/
|
||||
@@ -674,87 +515,8 @@ class Live2DChatActivity : AppCompatActivity() {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 从网络下载图片
|
||||
*/
|
||||
private fun downloadImage(url: String): Bitmap? {
|
||||
return try {
|
||||
// 使用与大模型相同的下载方式
|
||||
val tempFile = File(cacheDir, "temp_test_image_${System.currentTimeMillis()}.jpg")
|
||||
val success = FileHelper.downloadTestImage(url, tempFile)
|
||||
|
||||
if (success && tempFile.exists()) {
|
||||
val bitmap = BitmapFactory.decodeFile(tempFile.absolutePath)
|
||||
tempFile.delete() // 删除临时文件
|
||||
bitmap
|
||||
} else {
|
||||
Log.e(AppConfig.TAG, "Failed to download image: $url")
|
||||
null
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e(AppConfig.TAG, "Failed to download image: ${e.message}", e)
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测图片中的人脸
|
||||
*/
|
||||
private fun detectFace(bitmap: Bitmap): FaceBox? {
|
||||
Log.d(AppConfig.TAG, "[测试]Detecting face in bitmap: ${bitmap.width}x${bitmap.height}")
|
||||
return try {
|
||||
val engine = RetinaFaceEngineRKNN()
|
||||
Log.d(AppConfig.TAG, "[测试]Initializing RetinaFace engine...")
|
||||
if (engine.initialize(applicationContext)) {
|
||||
Log.d(AppConfig.TAG, "[测试]RetinaFace engine initialized successfully")
|
||||
val raw = engine.detect(bitmap)
|
||||
Log.d(AppConfig.TAG, "[测试]Face detection result: ${raw.joinToString(", ")}")
|
||||
engine.release()
|
||||
|
||||
if (raw.isNotEmpty()) {
|
||||
val stride = when {
|
||||
raw.size % 15 == 0 -> 15
|
||||
raw.size % 5 == 0 -> 5
|
||||
else -> 0
|
||||
}
|
||||
|
||||
Log.d(AppConfig.TAG, "[测试]Stride: $stride, raw size: ${raw.size}")
|
||||
|
||||
if (stride > 0) {
|
||||
val faceCount = raw.size / stride
|
||||
Log.d(AppConfig.TAG, "[测试]Detected $faceCount faces")
|
||||
if (faceCount > 0) {
|
||||
val i = 0
|
||||
val lm = if (stride >= 15) raw.copyOfRange(i + 5, i + 15) else null
|
||||
val hasLm = lm?.all { it >= 0f } == true
|
||||
|
||||
val faceBox = FaceBox(
|
||||
left = raw[i],
|
||||
top = raw[i + 1],
|
||||
right = raw[i + 2],
|
||||
bottom = raw[i + 3],
|
||||
score = raw[i + 4],
|
||||
hasLandmarks = hasLm,
|
||||
landmarks = if (hasLm) lm else null
|
||||
)
|
||||
Log.d(AppConfig.TAG, "[测试]Created face box: $faceBox")
|
||||
return faceBox
|
||||
}
|
||||
}
|
||||
} else {
|
||||
Log.w(AppConfig.TAG, "[测试]No faces detected in bitmap")
|
||||
}
|
||||
} else {
|
||||
Log.e(AppConfig.TAG, "[测试]Failed to initialize RetinaFace engine")
|
||||
}
|
||||
null
|
||||
} catch (e: Exception) {
|
||||
Log.e(AppConfig.TAG, "[测试]Failed to detect face: ${e.message}", e)
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
private fun createAsrCallback() = object : AsrManager.AsrCallback {
|
||||
override fun onAsrStarted() {
|
||||
currentTrace?.markASRStart()
|
||||
@@ -1247,27 +1009,30 @@ class Live2DChatActivity : AppCompatActivity() {
|
||||
Log.d(AppConfig.TAG, "Generated conversation summary for $activeUserId: $summary")
|
||||
}
|
||||
|
||||
// 使用 conversationBufferMemory 的对话记录提取用户信息
|
||||
// 使用多角度提问方式提取用户信息
|
||||
val dialogue = messages.joinToString("\n") { "${it.role}: ${it.content}" }
|
||||
|
||||
requestLocalProfileExtraction(dialogue) { raw ->
|
||||
requestMultiAngleProfileExtraction(dialogue) { profileData ->
|
||||
try {
|
||||
val json = parseFirstJsonObject(raw)
|
||||
val name = json.optString("name", "").trim().ifBlank { null }
|
||||
val age = json.optString("age", "").trim().ifBlank { null }
|
||||
val gender = json.optString("gender", "").trim().ifBlank { null }
|
||||
val hobbies = json.optString("hobbies", "").trim().ifBlank { null }
|
||||
val summary = json.optString("summary", "").trim().ifBlank { null }
|
||||
if (name != null) {
|
||||
userMemoryStore.updateDisplayName(activeUserId, name)
|
||||
}
|
||||
userMemoryStore.updateProfile(activeUserId, age, gender, hobbies, summary)
|
||||
|
||||
// 清空已处理的对话记录
|
||||
conversationBufferMemory.clear(activeUserId)
|
||||
|
||||
runOnUiThread {
|
||||
uiManager.appendToUi("\n[Memory] 已更新用户画像: $activeUserId\n")
|
||||
val nameToUpdate = profileData["name"]?.trim()?.ifBlank { null }
|
||||
val ageToUpdate = profileData["age"]?.trim()?.ifBlank { null }
|
||||
val genderToUpdate = profileData["gender"]?.trim()?.ifBlank { null }
|
||||
val hobbiesToUpdate = profileData["hobbies"]?.trim()?.ifBlank { null }
|
||||
val summaryToUpdate = profileData["summary"]?.trim()?.ifBlank { null }
|
||||
Log.d(TAG_LLM, "profileData: $profileData")
|
||||
if (nameToUpdate != null || ageToUpdate != null || genderToUpdate != null || hobbiesToUpdate != null || summaryToUpdate != null) {
|
||||
if (nameToUpdate != null) {
|
||||
userMemoryStore.updateDisplayName(activeUserId, nameToUpdate)
|
||||
Log.i(TAG_LLM, "Updated display name to $nameToUpdate")
|
||||
}
|
||||
userMemoryStore.updateProfile(activeUserId, ageToUpdate, genderToUpdate, hobbiesToUpdate, summaryToUpdate)
|
||||
|
||||
// 清空已处理的对话记录
|
||||
conversationBufferMemory.clear(activeUserId)
|
||||
|
||||
runOnUiThread {
|
||||
uiManager.appendToUi("\n[Memory] 已更新用户画像: $activeUserId\n")
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG_LLM, "Profile parse failed: ${e.message}")
|
||||
@@ -1275,27 +1040,77 @@ class Live2DChatActivity : AppCompatActivity() {
|
||||
}
|
||||
}
|
||||
|
||||
private fun requestLocalProfileExtraction(dialogue: String, onResult: (String) -> Unit) {
|
||||
private fun requestMultiAngleProfileExtraction(dialogue: String, onResult: (Map<String, String>) -> Unit) {
|
||||
try {
|
||||
val local = llmManager
|
||||
if (local == null) {
|
||||
onResult("{}")
|
||||
onResult(emptyMap())
|
||||
return
|
||||
}
|
||||
localThoughtSilentMode = true
|
||||
pendingLocalProfileCallback = onResult
|
||||
Log.i(TAG_LLM, "Routing profile extraction to LOCAL")
|
||||
local.generateResponseWithSystem(
|
||||
"你是信息抽取器。仅输出JSON对象，不要其他文字。字段为name,age,gender,hobbies,summary。",
|
||||
"请从以下对话提取用户信息，未知填空字符串，注意不需要：\n$dialogue"
|
||||
|
||||
val questions = listOf(
|
||||
"请从对话中提取用户的姓名，只返回姓名，如果没有提到姓名，请返回未知",
|
||||
"请从对话中提取用户的年龄，只返回年龄，如果没有提到年龄，请返回未知",
|
||||
"请从对话中提取用户的性别，只返回性别，如果没有提到性别，请返回未知",
|
||||
"请从对话中提取用户的爱好，只返回爱好，如果没有提到爱好，请返回未知",
|
||||
"请总结对话，只返回总结的内容"
|
||||
)
|
||||
|
||||
var completed = 0
|
||||
val results = mutableMapOf<String, String>()
|
||||
|
||||
questions.forEach { question ->
|
||||
val prompt = buildMultiAnglePrompt(dialogue, question)
|
||||
local.generate(prompt) { answer ->
|
||||
val processedAnswer = processProfileAnswer(answer)
|
||||
|
||||
when {
|
||||
question.contains("姓名") -> results["name"] = processedAnswer
|
||||
question.contains("年龄") -> results["age"] = processedAnswer
|
||||
question.contains("性别") -> results["gender"] = processedAnswer
|
||||
question.contains("爱好") -> results["hobbies"] = processedAnswer
|
||||
question.contains("总结") -> results["summary"] = processedAnswer
|
||||
}
|
||||
|
||||
completed++
|
||||
|
||||
if (completed == questions.size) {
|
||||
onResult(results)
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
pendingLocalProfileCallback = null
|
||||
localThoughtSilentMode = false
|
||||
Log.e(TAG_LLM, "requestLocalProfileExtraction failed: ${e.message}", e)
|
||||
onResult("{}")
|
||||
Log.e(TAG_LLM, "requestMultiAngleProfileExtraction failed: ${e.message}", e)
|
||||
onResult(emptyMap())
|
||||
}
|
||||
}
|
||||
|
||||
private fun buildMultiAnglePrompt(dialogue: String, question: String): String {
|
||||
return """
|
||||
请根据以下对话回答问题：
|
||||
|
||||
对话内容：
|
||||
$dialogue
|
||||
|
||||
问题：$question
|
||||
|
||||
回答：
|
||||
""".trimIndent()
|
||||
}
|
||||
|
||||
private fun processProfileAnswer(answer: String): String {
|
||||
var processed = answer.replace("<", "").replace(">", "")
|
||||
if (processed.contains("unknown", ignoreCase = true) ||
|
||||
processed.contains("null", ignoreCase = true) ||
|
||||
processed.contains("未知")) {
|
||||
return ""
|
||||
}
|
||||
if (processed.contains(":")) {
|
||||
processed = processed.substringAfter(":").trim()
|
||||
}
|
||||
processed = processed.replace(".", "").trim()
|
||||
return processed
|
||||
}
|
||||
|
||||
private fun parseFirstJsonObject(text: String): JSONObject {
|
||||
val raw = text.trim()
|
||||
@@ -1363,6 +1178,22 @@ class Live2DChatActivity : AppCompatActivity() {
|
||||
})
|
||||
Log.i(TAG_LLM, "LOCAL memory LLM initialized")
|
||||
useLocalLLM = true
|
||||
|
||||
if (AppConfig.OnboardTesting.LOCAL_LLM_SUMMARY) {
|
||||
llmSummaryTest = LLMSummaryTest(this)
|
||||
ioScope.launch {
|
||||
kotlinx.coroutines.delay(5000) // 等待5秒，确保LLMManager初始化完成
|
||||
runOnUiThread {
|
||||
if (llmManager != null) {
|
||||
llmSummaryTest.setLLMManager(llmManager!!)
|
||||
llmSummaryTest.runTest { message ->
|
||||
Log.i(AppConfig.TAG, message)
|
||||
uiManager.appendToUi("\n$message\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e(AppConfig.TAG, "Failed to initialize LLM: ${e.message}", e)
|
||||
Log.e(TAG_LLM, "LOCAL init failed: ${e.message}", e)
|
||||
|
||||
Reference in New Issue
Block a user