diff --git a/js/ai-teacher-app.js b/js/ai-teacher-app.js
index be86fef..a3aa71a 100644
--- a/js/ai-teacher-app.js
+++ b/js/ai-teacher-app.js
@@ -33,6 +33,14 @@ export class AITeacherApp {
this.currentSegmentIndex = 0;
this.isPlaying = false;
this.autoNavigate = true; // 默认启用自动翻页
+
+ // 音频上下文相关
+ this.audioContext = null;
+ this.audioSource = null;
+ this.analyzerNode = null;
+ this.audioContextInitialized = false;
+ this.lipSyncAnimationId = null;
+ this.frequencyData = null;
this.init();
}
@@ -345,13 +353,75 @@ export class AITeacherApp {
// 创建URL并设置到音频播放器
const audioUrl = URL.createObjectURL(blob);
- // self.Live2DController.playAudio(audioUrl)
this.audioPlayer.src = audioUrl;
this.audioPlayer.style.display = 'block';
// 高亮当前段中的句子
this.highlightCurrentSegmentSentences();
+ // 播放音频并启用口型同步
+ if (this.live2dController && this.live2dController.initialized) {
+ // 创建一个新的音频上下文
+ if (!this.audioContext) {
+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
+ }
+
+ // 创建媒体元素源
+ if (!this.audioSource) {
+ this.audioSource = this.audioContext.createMediaElementSource(this.audioPlayer);
+ }
+
+ // 启动音频分析
+ if (!this.analyzerNode) {
+ this.analyzerNode = this.audioContext.createAnalyser();
+ this.analyzerNode.fftSize = 256;
+ this.analyzerNode.smoothingTimeConstant = 0.3;
+ this.analyzerNode.minDecibels = -100;
+ this.analyzerNode.maxDecibels = -30;
+ }
+
+ // 连接音频源和分析器
+ this.audioSource.connect(this.analyzerNode);
+ this.analyzerNode.connect(this.audioContext.destination);
+
+ // 根据当前使用的模型选择合适的动作
+ const modelName = this.live2dController.model_setting.model;
+ let motionGroup = '';
+ let motionIndex = 0;
+
+ // 根据不同模型选择不同的动作组和索引
+ switch (modelName) {
+ case 'Haru':
+ motionGroup = 'haru_g_m';
+ motionIndex = Math.floor(Math.random() * 26) + 1; // 随机选择1-26之间的动作
+ break;
+ case 'Hiyori':
+ motionGroup = 'Hiyori_m';
+ motionIndex = Math.floor(Math.random() * 10) + 1; // 随机选择1-10之间的动作
+ break;
+ case 'Mao':
+ motionGroup = 'mtn_';
+ motionIndex = Math.floor(Math.random() * 4) + 1; // 随机选择1-4之间的动作
+ break;
+ case 'Mark':
+ motionGroup = 'mark_m';
+ motionIndex = Math.floor(Math.random() * 5) + 1; // 随机选择1-5之间的动作
+ break;
+ default:
+ // 如果没有匹配的模型，不播放动作
+ break;
+ }
+
+ // 如果有匹配的动作组，播放动作
+ if (motionGroup) {
+ try {
+ this.live2dController.playMotion(motionGroup, motionIndex);
+ } catch (error) {
+ console.log('播放动作时出错:', error);
+ }
+ }
+ }
+
// 播放音频
this.audioPlayer.play();
this.isPlaying = true;
@@ -364,10 +434,19 @@ export class AITeacherApp {
// 监听音频播放结束事件
this.audioPlayer.onended = () => {
+ // 结束口型同步
+ if (this.live2dController && this.live2dController.initialized) {
+ this.live2dController.setMouthOpenY(0);
+ }
+
// 播放下一段
this.currentSegmentIndex++;
this.playCurrentSegment();
};
+
+ // 启动口型同步动画
+ this.frequencyData = new Uint8Array(this.analyzerNode.frequencyBinCount);
+ this.lipSyncAnimationId = requestAnimationFrame(this.updateLipSync.bind(this));
} catch (error) {
console.error('播放音频时出错:', error);
this.showMessage('播放音频时出错: ' + error.message, true);
@@ -375,6 +454,244 @@ export class AITeacherApp {
}
}
+ pauseAudio() {
+ if (this.audioPlayer && !this.audioPlayer.paused) {
+ this.audioPlayer.pause();
+ this.isPlaying = false;
+
+ // 更新播放按钮状态
+ const playBtn = document.getElementById('play-btn');
+ playBtn.innerHTML = ' 播放';
+ playBtn.classList.remove('btn-warning');
+ playBtn.classList.add('btn-success');
+
+ // 暂停时重置口型
+ if (this.live2dController && this.live2dController.initialized) {
+ this.live2dController.setMouthOpenY(0);
+ }
+
+ // 暂停口型同步动画
+ if (this.lipSyncAnimationId) {
+ cancelAnimationFrame(this.lipSyncAnimationId);
+ this.lipSyncAnimationId = null;
+ }
+ } else if (this.audioPlayer && this.audioPlayer.paused) {
+ this.audioPlayer.play();
+ this.isPlaying = true;
+
+ // 更新播放按钮状态
+ const playBtn = document.getElementById('play-btn');
+ playBtn.innerHTML = ' 暂停';
+ playBtn.classList.remove('btn-success');
+ playBtn.classList.add('btn-warning');
+
+ // 启动口型同步动画
+ this.frequencyData = new Uint8Array(this.analyzerNode.frequencyBinCount);
+ this.lipSyncAnimationId = requestAnimationFrame(this.updateLipSync.bind(this));
+ }
+ }
+
+ resumeAudio() {
+ if (this.audioPlayer && this.audioPlayer.paused) {
+ this.audioPlayer.play();
+ this.isPlaying = true;
+
+ // 更新播放按钮状态
+ const playBtn = document.getElementById('play-btn');
+ playBtn.innerHTML = ' 暂停';
+ playBtn.classList.remove('btn-success');
+ playBtn.classList.add('btn-warning');
+
+ // 启动口型同步动画
+ this.frequencyData = new Uint8Array(this.analyzerNode.frequencyBinCount);
+ this.lipSyncAnimationId = requestAnimationFrame(this.updateLipSync.bind(this));
+ }
+ }
+
+ stopAudio() {
+ if (this.audioPlayer) {
+ this.audioPlayer.pause();
+ this.audioPlayer.currentTime = 0;
+ this.audioPlayer.style.display = 'none';
+ this.isPlaying = false;
+ this.currentSegmentIndex = 0;
+
+ // 更新播放按钮状态
+ const playBtn = document.getElementById('play-btn');
+ playBtn.innerHTML = ' 播放';
+ playBtn.classList.remove('btn-warning');
+ playBtn.classList.add('btn-success');
+
+ // 清除高亮
+ this.clearHighlights();
+
+ // 重置字幕
+ const subtitleText = document.getElementById('subtitle-text');
+ const subtitleContainer = document.getElementById('subtitle-container');
+
+ if (subtitleText) {
+ subtitleText.textContent = '字幕将在播放讲解时显示在这里';
+ subtitleContainer.classList.remove('active');
+ }
+
+ // 停止口型同步
+ if (this.live2dController && this.live2dController.initialized) {
+ this.live2dController.setMouthOpenY(0);
+
+ // 断开音频分析器连接
+ if (this.audioSource) {
+ try {
+ this.audioSource.disconnect();
+ } catch (error) {
+ console.log('断开音频源时出错:', error);
+ }
+ }
+
+ if (this.analyzerNode) {
+ try {
+ this.analyzerNode.disconnect();
+ } catch (error) {
+ console.log('断开分析器时出错:', error);
+ }
+ }
+
+ // 播放一个空闲动作
+ setTimeout(() => {
+ // 根据当前使用的模型选择合适的空闲动作
+ const modelName = this.live2dController.model_setting.model;
+
+ switch (modelName) {
+ case 'Haru':
+ this.live2dController.playMotion('haru_g_idle', 0);
+ break;
+ case 'Hiyori':
+ // Hiyori没有特定的idle动作，使用第一个动作
+ this.live2dController.playMotion('Hiyori_m', 1);
+ break;
+ case 'Mao':
+ // 对于Mao，使用sample_01作为空闲动作
+ this.live2dController.playMotion('sample_', 1);
+ break;
+ case 'Mark':
+ this.live2dController.playMotion('mark_m', 1);
+ break;
+ default:
+ // 如果没有匹配的模型，不播放动作
+ break;
+ }
+ }, 500);
+ }
+
+ // 停止口型同步动画
+ if (this.lipSyncAnimationId) {
+ cancelAnimationFrame(this.lipSyncAnimationId);
+ this.lipSyncAnimationId = null;
+ }
+ }
+ }
+
+ finishPlayback() {
+ // 播放完成后重置状态
+ this.isPlaying = false;
+ this.currentSegmentIndex = 0;
+
+ // 更新播放按钮状态
+ const playBtn = document.getElementById('play-btn');
+ playBtn.innerHTML = ' 播放';
+ playBtn.classList.remove('btn-warning');
+ playBtn.classList.add('btn-success');
+
+ // 清除高亮
+ this.clearHighlights();
+
+ // 重置字幕
+ const subtitleText = document.getElementById('subtitle-text');
+ const subtitleContainer = document.getElementById('subtitle-container');
+
+ if (subtitleText) {
+ subtitleText.textContent = '字幕将在播放讲解时显示在这里';
+ subtitleContainer.classList.remove('active');
+ }
+
+ // 停止口型同步
+ if (this.live2dController && this.live2dController.initialized) {
+ this.live2dController.setMouthOpenY(0);
+
+ // 断开音频分析器连接
+ if (this.audioSource) {
+ try {
+ this.audioSource.disconnect();
+ } catch (error) {
+ console.log('断开音频源时出错:', error);
+ }
+ }
+
+ if (this.analyzerNode) {
+ try {
+ this.analyzerNode.disconnect();
+ } catch (error) {
+ console.log('断开分析器时出错:', error);
+ }
+ }
+
+ // 播放一个空闲动作
+ setTimeout(() => {
+ // 根据当前使用的模型选择合适的空闲动作
+ const modelName = this.live2dController.model_setting.model;
+
+ switch (modelName) {
+ case 'Haru':
+ this.live2dController.playMotion('haru_g_idle', 0);
+ break;
+ case 'Hiyori':
+ // Hiyori没有特定的idle动作，使用第一个动作
+ this.live2dController.playMotion('Hiyori_m', 1);
+ break;
+ case 'Mao':
+ // 对于Mao，使用sample_01作为空闲动作
+ this.live2dController.playMotion('sample_', 1);
+ break;
+ case 'Mark':
+ this.live2dController.playMotion('mark_m', 1);
+ break;
+ default:
+ // 如果没有匹配的模型，不播放动作
+ break;
+ }
+ }, 500);
+ }
+
+ // 自动跳转到下一页（如果启用了自动翻页）
+ if (this.autoNavigate && this.pageNum < this.pdfDoc.numPages) {
+ // 延迟1.5秒后跳转，给用户一个短暂的停顿时间
+ setTimeout(() => {
+ this.onNextPage();
+ }, 1500);
+ }
+
+ // 停止口型同步动画
+ if (this.lipSyncAnimationId) {
+ cancelAnimationFrame(this.lipSyncAnimationId);
+ this.lipSyncAnimationId = null;
+ }
+ }
+
+ toggleAudio() {
+ if (!this.isPlaying) {
+ this.playAudio();
+ } else {
+ this.pauseAudio();
+ }
+ }
+
+ clearHighlights() {
+ // 清除所有句子的高亮
+ const sentences = document.querySelectorAll('.sentence');
+ sentences.forEach(sentence => {
+ sentence.classList.remove('highlight');
+ });
+ }
+
highlightCurrentSegmentSentences() {
// 清除所有高亮
this.clearHighlights();
@@ -426,106 +743,6 @@ export class AITeacherApp {
}
}
- pauseAudio() {
- if (this.audioPlayer && !this.audioPlayer.paused) {
- this.audioPlayer.pause();
- this.isPlaying = false;
-
- // 更新播放按钮状态
- const playBtn = document.getElementById('play-btn');
- playBtn.innerHTML = ' 播放';
- playBtn.classList.remove('btn-warning');
- playBtn.classList.add('btn-success');
- }
- }
-
- resumeAudio() {
- if (this.audioPlayer && this.audioPlayer.paused) {
- this.audioPlayer.play();
- this.isPlaying = true;
-
- // 更新播放按钮状态
- const playBtn = document.getElementById('play-btn');
- playBtn.innerHTML = ' 暂停';
- playBtn.classList.remove('btn-success');
- playBtn.classList.add('btn-warning');
- }
- }
-
- stopAudio() {
- if (this.audioPlayer) {
- this.audioPlayer.pause();
- this.audioPlayer.currentTime = 0;
- this.audioPlayer.style.display = 'none';
- this.isPlaying = false;
- this.currentSegmentIndex = 0;
-
- // 更新播放按钮状态
- const playBtn = document.getElementById('play-btn');
- playBtn.innerHTML = ' 播放';
- playBtn.classList.remove('btn-warning');
- playBtn.classList.add('btn-success');
-
- // 清除高亮
- this.clearHighlights();
-
- // 重置字幕
- const subtitleText = document.getElementById('subtitle-text');
- const subtitleContainer = document.getElementById('subtitle-container');
- if (subtitleText) {
- subtitleText.textContent = '字幕将在播放讲解时显示在这里';
- subtitleContainer.classList.remove('active');
- }
- }
- }
-
- finishPlayback() {
- // 播放完成后重置状态
- this.isPlaying = false;
- this.currentSegmentIndex = 0;
-
- // 更新播放按钮状态
- const playBtn = document.getElementById('play-btn');
- playBtn.innerHTML = ' 播放';
- playBtn.classList.remove('btn-warning');
- playBtn.classList.add('btn-success');
-
- // 清除高亮
- this.clearHighlights();
-
- // 重置字幕
- const subtitleText = document.getElementById('subtitle-text');
- const subtitleContainer = document.getElementById('subtitle-container');
- if (subtitleText) {
- subtitleText.textContent = '字幕将在播放讲解时显示在这里';
- subtitleContainer.classList.remove('active');
- }
-
- // 自动跳转到下一页（如果启用了自动翻页）
- if (this.autoNavigate && this.pageNum < this.pdfDoc.numPages) {
- // 延迟1.5秒后跳转，给用户一个短暂的停顿时间
- setTimeout(() => {
- this.onNextPage();
- }, 1500);
- }
- }
-
- toggleAudio() {
- if (!this.isPlaying) {
- this.playAudio();
- } else {
- this.pauseAudio();
- }
- }
-
- clearHighlights() {
- // 清除所有句子的高亮
- const sentences = document.querySelectorAll('.sentence');
- sentences.forEach(sentence => {
- sentence.classList.remove('highlight');
- });
- }
-
initVoiceControls() {
// 初始化音频播放器
this.audioPlayer = document.getElementById('audio-player');
@@ -613,4 +830,43 @@ export class AITeacherApp {
// 尝试加载可用的语音列表
this.loadVoices();
}
+
+ updateLipSync() {
+ if (!this.analyzerNode || !this.frequencyData || !this.isPlaying) {
+ return;
+ }
+
+ this.analyzerNode.getByteFrequencyData(this.frequencyData);
+
+ // 人声主要集中在85-255Hz范围内
+ // 我们的FFT大小是256，所以我们需要计算这个范围对应的索引
+ const minFreqIndex = Math.floor(85 / (this.audioContext.sampleRate / 2 / 128));
+ const maxFreqIndex = Math.floor(255 / (this.audioContext.sampleRate / 2 / 128));
+
+ // 计算语音频率范围内的平均值
+ let sum = 0;
+ let count = 0;
+
+ for (let i = minFreqIndex; i <= maxFreqIndex && i < this.frequencyData.length; i++) {
+ sum += this.frequencyData[i];
+ count++;
+ }
+
+ const average = count > 0 ? sum / count : 0;
+
+ // 应用平滑处理使口型动画更自然
+ const normalizedValue = Math.min(1, Math.max(0, average / 200));
+
+ // 添加一些随机变化使口型看起来更自然
+ const jitter = Math.random() * 0.05;
+ const finalValue = normalizedValue * (0.95 + jitter);
+
+ // 根据平均值更新口型
+ if (this.live2dController && this.live2dController.initialized) {
+ this.live2dController.setMouthOpenY(finalValue);
+ }
+
+ // 继续更新口型
+ this.lipSyncAnimationId = requestAnimationFrame(this.updateLipSync.bind(this));
+ }
}