debug
This commit is contained in:
turn_wind
parent
8c599fc9e4
commit
19c37f1812
@ -34,6 +34,14 @@ export class AITeacherApp {
|
||||
this.isPlaying = false;
|
||||
this.autoNavigate = true; // 默认启用自动翻页
|
||||
|
||||
// 音频上下文相关
|
||||
this.audioContext = null;
|
||||
this.audioSource = null;
|
||||
this.analyzerNode = null;
|
||||
this.audioContextInitialized = false;
|
||||
this.lipSyncAnimationId = null;
|
||||
this.frequencyData = null;
|
||||
|
||||
this.init();
|
||||
}
|
||||
|
||||
@ -345,13 +353,75 @@ export class AITeacherApp {
|
||||
|
||||
// 创建URL并设置到音频播放器
|
||||
const audioUrl = URL.createObjectURL(blob);
|
||||
// self.Live2DController.playAudio(audioUrl)
|
||||
this.audioPlayer.src = audioUrl;
|
||||
this.audioPlayer.style.display = 'block';
|
||||
|
||||
// 高亮当前段中的句子
|
||||
this.highlightCurrentSegmentSentences();
|
||||
|
||||
// 播放音频并启用口型同步
|
||||
if (this.live2dController && this.live2dController.initialized) {
|
||||
// 创建一个新的音频上下文
|
||||
if (!this.audioContext) {
|
||||
this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
|
||||
}
|
||||
|
||||
// 创建媒体元素源
|
||||
if (!this.audioSource) {
|
||||
this.audioSource = this.audioContext.createMediaElementSource(this.audioPlayer);
|
||||
}
|
||||
|
||||
// 启动音频分析
|
||||
if (!this.analyzerNode) {
|
||||
this.analyzerNode = this.audioContext.createAnalyser();
|
||||
this.analyzerNode.fftSize = 256;
|
||||
this.analyzerNode.smoothingTimeConstant = 0.3;
|
||||
this.analyzerNode.minDecibels = -100;
|
||||
this.analyzerNode.maxDecibels = -30;
|
||||
}
|
||||
|
||||
// 连接音频源和分析器
|
||||
this.audioSource.connect(this.analyzerNode);
|
||||
this.analyzerNode.connect(this.audioContext.destination);
|
||||
|
||||
// 根据当前使用的模型选择合适的动作
|
||||
const modelName = this.live2dController.model_setting.model;
|
||||
let motionGroup = '';
|
||||
let motionIndex = 0;
|
||||
|
||||
// 根据不同模型选择不同的动作组和索引
|
||||
switch (modelName) {
|
||||
case 'Haru':
|
||||
motionGroup = 'haru_g_m';
|
||||
motionIndex = Math.floor(Math.random() * 26) + 1; // 随机选择1-26之间的动作
|
||||
break;
|
||||
case 'Hiyori':
|
||||
motionGroup = 'Hiyori_m';
|
||||
motionIndex = Math.floor(Math.random() * 10) + 1; // 随机选择1-10之间的动作
|
||||
break;
|
||||
case 'Mao':
|
||||
motionGroup = 'mtn_';
|
||||
motionIndex = Math.floor(Math.random() * 4) + 1; // 随机选择1-4之间的动作
|
||||
break;
|
||||
case 'Mark':
|
||||
motionGroup = 'mark_m';
|
||||
motionIndex = Math.floor(Math.random() * 5) + 1; // 随机选择1-5之间的动作
|
||||
break;
|
||||
default:
|
||||
// 如果没有匹配的模型，不播放动作
|
||||
break;
|
||||
}
|
||||
|
||||
// 如果有匹配的动作组，播放动作
|
||||
if (motionGroup) {
|
||||
try {
|
||||
this.live2dController.playMotion(motionGroup, motionIndex);
|
||||
} catch (error) {
|
||||
console.log('播放动作时出错:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 播放音频
|
||||
this.audioPlayer.play();
|
||||
this.isPlaying = true;
|
||||
@ -364,10 +434,19 @@ export class AITeacherApp {
|
||||
|
||||
// 监听音频播放结束事件
|
||||
this.audioPlayer.onended = () => {
|
||||
// 结束口型同步
|
||||
if (this.live2dController && this.live2dController.initialized) {
|
||||
this.live2dController.setMouthOpenY(0);
|
||||
}
|
||||
|
||||
// 播放下一段
|
||||
this.currentSegmentIndex++;
|
||||
this.playCurrentSegment();
|
||||
};
|
||||
|
||||
// 启动口型同步动画
|
||||
this.frequencyData = new Uint8Array(this.analyzerNode.frequencyBinCount);
|
||||
this.lipSyncAnimationId = requestAnimationFrame(this.updateLipSync.bind(this));
|
||||
} catch (error) {
|
||||
console.error('播放音频时出错:', error);
|
||||
this.showMessage('播放音频时出错: ' + error.message, true);
|
||||
@ -375,6 +454,244 @@ export class AITeacherApp {
|
||||
}
|
||||
}
|
||||
|
||||
pauseAudio() {
|
||||
if (this.audioPlayer && !this.audioPlayer.paused) {
|
||||
this.audioPlayer.pause();
|
||||
this.isPlaying = false;
|
||||
|
||||
// 更新播放按钮状态
|
||||
const playBtn = document.getElementById('play-btn');
|
||||
playBtn.innerHTML = '<i class="bi bi-play-fill"></i> 播放';
|
||||
playBtn.classList.remove('btn-warning');
|
||||
playBtn.classList.add('btn-success');
|
||||
|
||||
// 暂停时重置口型
|
||||
if (this.live2dController && this.live2dController.initialized) {
|
||||
this.live2dController.setMouthOpenY(0);
|
||||
}
|
||||
|
||||
// 暂停口型同步动画
|
||||
if (this.lipSyncAnimationId) {
|
||||
cancelAnimationFrame(this.lipSyncAnimationId);
|
||||
this.lipSyncAnimationId = null;
|
||||
}
|
||||
} else if (this.audioPlayer && this.audioPlayer.paused) {
|
||||
this.audioPlayer.play();
|
||||
this.isPlaying = true;
|
||||
|
||||
// 更新播放按钮状态
|
||||
const playBtn = document.getElementById('play-btn');
|
||||
playBtn.innerHTML = '<i class="bi bi-pause-fill"></i> 暂停';
|
||||
playBtn.classList.remove('btn-success');
|
||||
playBtn.classList.add('btn-warning');
|
||||
|
||||
// 启动口型同步动画
|
||||
this.frequencyData = new Uint8Array(this.analyzerNode.frequencyBinCount);
|
||||
this.lipSyncAnimationId = requestAnimationFrame(this.updateLipSync.bind(this));
|
||||
}
|
||||
}
|
||||
|
||||
resumeAudio() {
|
||||
if (this.audioPlayer && this.audioPlayer.paused) {
|
||||
this.audioPlayer.play();
|
||||
this.isPlaying = true;
|
||||
|
||||
// 更新播放按钮状态
|
||||
const playBtn = document.getElementById('play-btn');
|
||||
playBtn.innerHTML = '<i class="bi bi-pause-fill"></i> 暂停';
|
||||
playBtn.classList.remove('btn-success');
|
||||
playBtn.classList.add('btn-warning');
|
||||
|
||||
// 启动口型同步动画
|
||||
this.frequencyData = new Uint8Array(this.analyzerNode.frequencyBinCount);
|
||||
this.lipSyncAnimationId = requestAnimationFrame(this.updateLipSync.bind(this));
|
||||
}
|
||||
}
|
||||
|
||||
stopAudio() {
|
||||
if (this.audioPlayer) {
|
||||
this.audioPlayer.pause();
|
||||
this.audioPlayer.currentTime = 0;
|
||||
this.audioPlayer.style.display = 'none';
|
||||
this.isPlaying = false;
|
||||
this.currentSegmentIndex = 0;
|
||||
|
||||
// 更新播放按钮状态
|
||||
const playBtn = document.getElementById('play-btn');
|
||||
playBtn.innerHTML = '<i class="bi bi-play-fill"></i> 播放';
|
||||
playBtn.classList.remove('btn-warning');
|
||||
playBtn.classList.add('btn-success');
|
||||
|
||||
// 清除高亮
|
||||
this.clearHighlights();
|
||||
|
||||
// 重置字幕
|
||||
const subtitleText = document.getElementById('subtitle-text');
|
||||
const subtitleContainer = document.getElementById('subtitle-container');
|
||||
|
||||
if (subtitleText) {
|
||||
subtitleText.textContent = '字幕将在播放讲解时显示在这里';
|
||||
subtitleContainer.classList.remove('active');
|
||||
}
|
||||
|
||||
// 停止口型同步
|
||||
if (this.live2dController && this.live2dController.initialized) {
|
||||
this.live2dController.setMouthOpenY(0);
|
||||
|
||||
// 断开音频分析器连接
|
||||
if (this.audioSource) {
|
||||
try {
|
||||
this.audioSource.disconnect();
|
||||
} catch (error) {
|
||||
console.log('断开音频源时出错:', error);
|
||||
}
|
||||
}
|
||||
|
||||
if (this.analyzerNode) {
|
||||
try {
|
||||
this.analyzerNode.disconnect();
|
||||
} catch (error) {
|
||||
console.log('断开分析器时出错:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// 播放一个空闲动作
|
||||
setTimeout(() => {
|
||||
// 根据当前使用的模型选择合适的空闲动作
|
||||
const modelName = this.live2dController.model_setting.model;
|
||||
|
||||
switch (modelName) {
|
||||
case 'Haru':
|
||||
this.live2dController.playMotion('haru_g_idle', 0);
|
||||
break;
|
||||
case 'Hiyori':
|
||||
// Hiyori没有特定的idle动作，使用第一个动作
|
||||
this.live2dController.playMotion('Hiyori_m', 1);
|
||||
break;
|
||||
case 'Mao':
|
||||
// 对于Mao，使用sample_01作为空闲动作
|
||||
this.live2dController.playMotion('sample_', 1);
|
||||
break;
|
||||
case 'Mark':
|
||||
this.live2dController.playMotion('mark_m', 1);
|
||||
break;
|
||||
default:
|
||||
// 如果没有匹配的模型，不播放动作
|
||||
break;
|
||||
}
|
||||
}, 500);
|
||||
}
|
||||
|
||||
// 停止口型同步动画
|
||||
if (this.lipSyncAnimationId) {
|
||||
cancelAnimationFrame(this.lipSyncAnimationId);
|
||||
this.lipSyncAnimationId = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
finishPlayback() {
|
||||
// 播放完成后重置状态
|
||||
this.isPlaying = false;
|
||||
this.currentSegmentIndex = 0;
|
||||
|
||||
// 更新播放按钮状态
|
||||
const playBtn = document.getElementById('play-btn');
|
||||
playBtn.innerHTML = '<i class="bi bi-play-fill"></i> 播放';
|
||||
playBtn.classList.remove('btn-warning');
|
||||
playBtn.classList.add('btn-success');
|
||||
|
||||
// 清除高亮
|
||||
this.clearHighlights();
|
||||
|
||||
// 重置字幕
|
||||
const subtitleText = document.getElementById('subtitle-text');
|
||||
const subtitleContainer = document.getElementById('subtitle-container');
|
||||
|
||||
if (subtitleText) {
|
||||
subtitleText.textContent = '字幕将在播放讲解时显示在这里';
|
||||
subtitleContainer.classList.remove('active');
|
||||
}
|
||||
|
||||
// 停止口型同步
|
||||
if (this.live2dController && this.live2dController.initialized) {
|
||||
this.live2dController.setMouthOpenY(0);
|
||||
|
||||
// 断开音频分析器连接
|
||||
if (this.audioSource) {
|
||||
try {
|
||||
this.audioSource.disconnect();
|
||||
} catch (error) {
|
||||
console.log('断开音频源时出错:', error);
|
||||
}
|
||||
}
|
||||
|
||||
if (this.analyzerNode) {
|
||||
try {
|
||||
this.analyzerNode.disconnect();
|
||||
} catch (error) {
|
||||
console.log('断开分析器时出错:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// 播放一个空闲动作
|
||||
setTimeout(() => {
|
||||
// 根据当前使用的模型选择合适的空闲动作
|
||||
const modelName = this.live2dController.model_setting.model;
|
||||
|
||||
switch (modelName) {
|
||||
case 'Haru':
|
||||
this.live2dController.playMotion('haru_g_idle', 0);
|
||||
break;
|
||||
case 'Hiyori':
|
||||
// Hiyori没有特定的idle动作，使用第一个动作
|
||||
this.live2dController.playMotion('Hiyori_m', 1);
|
||||
break;
|
||||
case 'Mao':
|
||||
// 对于Mao，使用sample_01作为空闲动作
|
||||
this.live2dController.playMotion('sample_', 1);
|
||||
break;
|
||||
case 'Mark':
|
||||
this.live2dController.playMotion('mark_m', 1);
|
||||
break;
|
||||
default:
|
||||
// 如果没有匹配的模型，不播放动作
|
||||
break;
|
||||
}
|
||||
}, 500);
|
||||
}
|
||||
|
||||
// 自动跳转到下一页（如果启用了自动翻页）
|
||||
if (this.autoNavigate && this.pageNum < this.pdfDoc.numPages) {
|
||||
// 延迟1.5秒后跳转，给用户一个短暂的停顿时间
|
||||
setTimeout(() => {
|
||||
this.onNextPage();
|
||||
}, 1500);
|
||||
}
|
||||
|
||||
// 停止口型同步动画
|
||||
if (this.lipSyncAnimationId) {
|
||||
cancelAnimationFrame(this.lipSyncAnimationId);
|
||||
this.lipSyncAnimationId = null;
|
||||
}
|
||||
}
|
||||
|
||||
toggleAudio() {
|
||||
if (!this.isPlaying) {
|
||||
this.playAudio();
|
||||
} else {
|
||||
this.pauseAudio();
|
||||
}
|
||||
}
|
||||
|
||||
clearHighlights() {
|
||||
// 清除所有句子的高亮
|
||||
const sentences = document.querySelectorAll('.sentence');
|
||||
sentences.forEach(sentence => {
|
||||
sentence.classList.remove('highlight');
|
||||
});
|
||||
}
|
||||
|
||||
highlightCurrentSegmentSentences() {
|
||||
// 清除所有高亮
|
||||
this.clearHighlights();
|
||||
@ -426,106 +743,6 @@ export class AITeacherApp {
|
||||
}
|
||||
}
|
||||
|
||||
pauseAudio() {
|
||||
if (this.audioPlayer && !this.audioPlayer.paused) {
|
||||
this.audioPlayer.pause();
|
||||
this.isPlaying = false;
|
||||
|
||||
// 更新播放按钮状态
|
||||
const playBtn = document.getElementById('play-btn');
|
||||
playBtn.innerHTML = '<i class="bi bi-play-fill"></i> 播放';
|
||||
playBtn.classList.remove('btn-warning');
|
||||
playBtn.classList.add('btn-success');
|
||||
}
|
||||
}
|
||||
|
||||
resumeAudio() {
|
||||
if (this.audioPlayer && this.audioPlayer.paused) {
|
||||
this.audioPlayer.play();
|
||||
this.isPlaying = true;
|
||||
|
||||
// 更新播放按钮状态
|
||||
const playBtn = document.getElementById('play-btn');
|
||||
playBtn.innerHTML = '<i class="bi bi-pause-fill"></i> 暂停';
|
||||
playBtn.classList.remove('btn-success');
|
||||
playBtn.classList.add('btn-warning');
|
||||
}
|
||||
}
|
||||
|
||||
stopAudio() {
|
||||
if (this.audioPlayer) {
|
||||
this.audioPlayer.pause();
|
||||
this.audioPlayer.currentTime = 0;
|
||||
this.audioPlayer.style.display = 'none';
|
||||
this.isPlaying = false;
|
||||
this.currentSegmentIndex = 0;
|
||||
|
||||
// 更新播放按钮状态
|
||||
const playBtn = document.getElementById('play-btn');
|
||||
playBtn.innerHTML = '<i class="bi bi-play-fill"></i> 播放';
|
||||
playBtn.classList.remove('btn-warning');
|
||||
playBtn.classList.add('btn-success');
|
||||
|
||||
// 清除高亮
|
||||
this.clearHighlights();
|
||||
|
||||
// 重置字幕
|
||||
const subtitleText = document.getElementById('subtitle-text');
|
||||
const subtitleContainer = document.getElementById('subtitle-container');
|
||||
if (subtitleText) {
|
||||
subtitleText.textContent = '字幕将在播放讲解时显示在这里';
|
||||
subtitleContainer.classList.remove('active');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
finishPlayback() {
|
||||
// 播放完成后重置状态
|
||||
this.isPlaying = false;
|
||||
this.currentSegmentIndex = 0;
|
||||
|
||||
// 更新播放按钮状态
|
||||
const playBtn = document.getElementById('play-btn');
|
||||
playBtn.innerHTML = '<i class="bi bi-play-fill"></i> 播放';
|
||||
playBtn.classList.remove('btn-warning');
|
||||
playBtn.classList.add('btn-success');
|
||||
|
||||
// 清除高亮
|
||||
this.clearHighlights();
|
||||
|
||||
// 重置字幕
|
||||
const subtitleText = document.getElementById('subtitle-text');
|
||||
const subtitleContainer = document.getElementById('subtitle-container');
|
||||
if (subtitleText) {
|
||||
subtitleText.textContent = '字幕将在播放讲解时显示在这里';
|
||||
subtitleContainer.classList.remove('active');
|
||||
}
|
||||
|
||||
// 自动跳转到下一页（如果启用了自动翻页）
|
||||
if (this.autoNavigate && this.pageNum < this.pdfDoc.numPages) {
|
||||
// 延迟1.5秒后跳转，给用户一个短暂的停顿时间
|
||||
setTimeout(() => {
|
||||
this.onNextPage();
|
||||
}, 1500);
|
||||
}
|
||||
}
|
||||
|
||||
toggleAudio() {
|
||||
if (!this.isPlaying) {
|
||||
this.playAudio();
|
||||
} else {
|
||||
this.pauseAudio();
|
||||
}
|
||||
}
|
||||
|
||||
clearHighlights() {
|
||||
// 清除所有句子的高亮
|
||||
const sentences = document.querySelectorAll('.sentence');
|
||||
sentences.forEach(sentence => {
|
||||
sentence.classList.remove('highlight');
|
||||
});
|
||||
}
|
||||
|
||||
initVoiceControls() {
|
||||
// 初始化音频播放器
|
||||
this.audioPlayer = document.getElementById('audio-player');
|
||||
@ -613,4 +830,43 @@ export class AITeacherApp {
|
||||
// 尝试加载可用的语音列表
|
||||
this.loadVoices();
|
||||
}
|
||||
|
||||
updateLipSync() {
|
||||
if (!this.analyzerNode || !this.frequencyData || !this.isPlaying) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.analyzerNode.getByteFrequencyData(this.frequencyData);
|
||||
|
||||
// 人声主要集中在85-255Hz范围内
|
||||
// 我们的FFT大小是256，所以我们需要计算这个范围对应的索引
|
||||
const minFreqIndex = Math.floor(85 / (this.audioContext.sampleRate / 2 / 128));
|
||||
const maxFreqIndex = Math.floor(255 / (this.audioContext.sampleRate / 2 / 128));
|
||||
|
||||
// 计算语音频率范围内的平均值
|
||||
let sum = 0;
|
||||
let count = 0;
|
||||
|
||||
for (let i = minFreqIndex; i <= maxFreqIndex && i < this.frequencyData.length; i++) {
|
||||
sum += this.frequencyData[i];
|
||||
count++;
|
||||
}
|
||||
|
||||
const average = count > 0 ? sum / count : 0;
|
||||
|
||||
// 应用平滑处理使口型动画更自然
|
||||
const normalizedValue = Math.min(1, Math.max(0, average / 200));
|
||||
|
||||
// 添加一些随机变化使口型看起来更自然
|
||||
const jitter = Math.random() * 0.05;
|
||||
const finalValue = normalizedValue * (0.95 + jitter);
|
||||
|
||||
// 根据平均值更新口型
|
||||
if (this.live2dController && this.live2dController.initialized) {
|
||||
this.live2dController.setMouthOpenY(finalValue);
|
||||
}
|
||||
|
||||
// 继续更新口型
|
||||
this.lipSyncAnimationId = requestAnimationFrame(this.updateLipSync.bind(this));
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user