添加: 音频视频分开解码

AvPlayer/ffmpegvideopuller.cpp

@@ -102,13 +102,21 @@ void FFmpegVideoPuller::start()
         return;
     m_running = true;
 
+    // 初始化PTS
+    m_videoPts = 0.0;
+    m_audioPts = 0.0;
+
     // 解码线程
     m_decodeThread = QThread::create([this]() { this->decodeLoop(); });
     m_decodeThread->start();
 
-    // 播放线程
-    m_playThread = QThread::create([this]() { this->playLoop(); });
-    m_playThread->start();
+    // 视频播放线程
+    m_videoPlayThread = QThread::create([this]() { this->videoPlayLoop(); });
+    m_videoPlayThread->start();
+
+    // 音频播放线程
+    m_audioPlayThread = QThread::create([this]() { this->audioPlayLoop(); });
+    m_audioPlayThread->start();
 }
 
 void FFmpegVideoPuller::stop()
@@ -120,11 +128,17 @@ void FFmpegVideoPuller::stop()
         delete m_decodeThread;
         m_decodeThread = nullptr;
     }
-    if (m_playThread) {
-        m_playThread->quit();
-        m_playThread->wait();
-        delete m_playThread;
-        m_playThread = nullptr;
+    if (m_videoPlayThread) {
+        m_videoPlayThread->quit();
+        m_videoPlayThread->wait();
+        delete m_videoPlayThread;
+        m_videoPlayThread = nullptr;
+    }
+    if (m_audioPlayThread) {
+        m_audioPlayThread->quit();
+        m_audioPlayThread->wait();
+        delete m_audioPlayThread;
+        m_audioPlayThread = nullptr;
     }
     // 释放缓冲区
     if (m_videoBuffer) {
@@ -178,61 +192,29 @@ void FFmpegVideoPuller::decodeLoop()
     av_packet_free(&pkt);
 }
 
-void FFmpegVideoPuller::playLoop()
+void FFmpegVideoPuller::videoPlayLoop()
 {
-    // while (m_running) {
-    //     AVFrame* vFrame = getCurrentVideoFrame();
-    //     if (!vFrame) {
-    //         std::this_thread::sleep_for(std::chrono::milliseconds(5));
-    //         continue;
-    //     }
-    //     double vPts = vFrame->best_effort_timestamp
-    //                   * av_q2d(m_fmtCtx->streams[m_videoStreamIdx]->time_base);
-
-    //     // 这里你可以直接把vFrame送到OpenGL渲染
-    //     // 例如: renderFrame(vFrame);
-
-    //     // 控制倍速
-    //     AVRational fr = m_fmtCtx->streams[m_videoStreamIdx]->avg_frame_rate;
-    //     double frame_delay = 1.0 / (fr.num ? av_q2d(fr) : 25.0);
-    //     float speed = getSpeed();
-    //     int delay = int(frame_delay * 1000 / speed);
-    //     qDebug() << "playLoop, m_speed=" << speed << delay;
-    //     if (delay > 0)
-    //         std::this_thread::sleep_for(std::chrono::milliseconds(delay));
-
-    //     // 前进
-    //     nextVideoFrame();
+    // 如果没有视频流，直接返回
+    if (m_videoStreamIdx == -1) {
+        return;
+    }
 
-    //     // 更新当前pts
-    //     {
-    //         std::lock_guard<std::mutex> lock(m_ptsMutex);
-    //         m_currentPts = vPts;
-    //     }
-    // }
     while (m_running) {
         AVFrame* vFrame = getCurrentVideoFrame();
-        double vPts = vFrame ? (vFrame->best_effort_timestamp
-                                * av_q2d(m_fmtCtx->streams[m_videoStreamIdx]->time_base))
-                             : m_currentPts.load();
-        if (vFrame && m_videoRenderCallback) {
-            m_videoRenderCallback(vFrame);
+        if (!vFrame) {
+            std::this_thread::sleep_for(std::chrono::milliseconds(5));
+            continue;
         }
-
-        // 音频同步：只要音频帧PTS小于等于视频帧PTS，就顺序播放
-        while (true) {
-            AVFrame* aFrame = getCurrentAudioFrame();
-            if (!aFrame)
-                break;
-            double aPts = aFrame->best_effort_timestamp
-                          * av_q2d(m_fmtCtx->streams[m_audioStreamIdx]->time_base);
-            if (aPts <= vPts + 0.02) { // 允许微小误差
-                if (m_audioPlayCallback)
-                    m_audioPlayCallback(aFrame);
-                nextAudioFrame();
-            } else {
-                break;
-            }
+        
+        double vPts = vFrame->best_effort_timestamp
+                      * av_q2d(m_fmtCtx->streams[m_videoStreamIdx]->time_base);
+        
+        // 更新视频PTS
+        m_videoPts = vPts;
+        
+        // 渲染视频帧
+        if (m_videoRenderCallback) {
+            m_videoRenderCallback(vFrame);
         }
 
         // 控制倍速
@@ -240,11 +222,32 @@ void FFmpegVideoPuller::playLoop()
         double frame_delay = 1.0 / (fr.num ? av_q2d(fr) : 25.0);
         float speed = getSpeed();
         int delay = int(frame_delay * 1000 / speed);
-        if (delay > 0)
-            std::this_thread::sleep_for(std::chrono::milliseconds(delay));
-        // 前进
+        
+        // 如果只有视频没有音频，直接按照视频帧率播放
+        if (m_audioStreamIdx == -1) {
+            if (delay > 0) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(delay));
+            }
+        } else {
+            // 有音频时，需要考虑音视频同步
+            double aPts = m_audioPts.load();
+            
+            // 如果视频比音频快太多，等待音频追上
+            if (vPts > aPts + 0.1) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(10));
+                continue; // 不前进到下一帧，等待音频追上
+            }
+            
+            // 正常播放速度控制
+            if (delay > 0) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(delay));
+            }
+        }
+        
+        // 前进到下一帧
         nextVideoFrame();
-        // 更新当前pts
+        
+        // 更新当前pts（用于进度显示）
         {
             std::lock_guard<std::mutex> lock(m_ptsMutex);
             m_currentPts = vPts;
@@ -252,6 +255,96 @@ void FFmpegVideoPuller::playLoop()
     }
 }
 
+void FFmpegVideoPuller::audioPlayLoop()
+{
+    // 如果没有音频流，直接返回
+    if (m_audioStreamIdx == -1) {
+        return;
+    }
+    
+    while (m_running) {
+        // 获取当前音频帧
+        AVFrame* aFrame = getCurrentAudioFrame();
+        if (!aFrame) {
+            std::this_thread::sleep_for(std::chrono::milliseconds(5));
+            continue;
+        }
+        
+        // 计算音频PTS
+        double aPts = aFrame->best_effort_timestamp
+                      * av_q2d(m_fmtCtx->streams[m_audioStreamIdx]->time_base);
+        
+        // 更新音频PTS
+        m_audioPts = aPts;
+        
+        // 如果只有音频没有视频，直接播放音频
+        if (m_videoStreamIdx == -1) {
+            // 播放音频帧
+            if (m_audioPlayCallback) {
+                m_audioPlayCallback(aFrame);
+            }
+            
+            // 更新当前pts（用于进度显示）
+            {
+                std::lock_guard<std::mutex> lock(m_ptsMutex);
+                m_currentPts = aPts;
+            }
+            
+            // 前进到下一帧
+            nextAudioFrame();
+            
+            // 控制音频播放速度
+            float speed = getSpeed();
+            int samples = aFrame->nb_samples;
+            int sample_rate = aFrame->sample_rate;
+            double frame_duration = samples / (double)sample_rate;
+            int delay = int(frame_duration * 1000 / speed);
+            if (delay > 0) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(delay));
+            }
+            
+            continue; // 继续下一帧处理
+        }
+        
+        // 有视频时，需要考虑音视频同步
+        double vPts = m_videoPts.load();
+        
+        // 如果音频PTS超前于视频PTS太多，等待视频追上
+        if (aPts > vPts + 0.1) {
+            std::this_thread::sleep_for(std::chrono::milliseconds(5));
+            continue; // 不前进到下一帧，等待视频追上
+        }
+        
+        // 如果音频比视频慢太多，可能需要跳过一些音频帧来追赶视频
+        if (aPts < vPts - 0.2) {
+            // 跳过当前音频帧，直接前进
+            nextAudioFrame();
+            continue;
+        }
+        
+        // 播放音频帧
+        if (m_audioPlayCallback) {
+            m_audioPlayCallback(aFrame);
+        }
+        
+        // 前进到下一帧
+        nextAudioFrame();
+        
+        // 控制音频播放速度，与视频保持一致
+        float speed = getSpeed();
+        if (speed != 1.0f) {
+            // 根据倍速调整音频播放间隔
+            int samples = aFrame->nb_samples;
+            int sample_rate = aFrame->sample_rate;
+            double frame_duration = samples / (double)sample_rate;
+            int delay = int(frame_duration * 1000 / speed);
+            if (delay > 0) {
+                std::this_thread::sleep_for(std::chrono::milliseconds(delay));
+            }
+        }
+    }
+}
+
 // 进度相关
 double FFmpegVideoPuller::getFirstPts() const
 {
@@ -271,24 +364,36 @@ void FFmpegVideoPuller::seekToPts(double pts)
     // 停止线程
     m_running = false;
     if (m_decodeThread) { m_decodeThread->quit(); m_decodeThread->wait(); delete m_decodeThread; m_decodeThread = nullptr; }
-    if (m_playThread) { m_playThread->quit(); m_playThread->wait(); delete m_playThread; m_playThread = nullptr; }
+    if (m_videoPlayThread) { m_videoPlayThread->quit(); m_videoPlayThread->wait(); delete m_videoPlayThread; m_videoPlayThread = nullptr; }
+    if (m_audioPlayThread) { m_audioPlayThread->quit(); m_audioPlayThread->wait(); delete m_audioPlayThread; m_audioPlayThread = nullptr; }
+    
     // 清空缓冲区
     if (m_videoBuffer) m_videoBuffer->clear();
     if (m_audioBuffer) m_audioBuffer->clear();
+    
     // FFmpeg seek
     int64_t seek_target = pts / av_q2d(m_fmtCtx->streams[m_videoStreamIdx]->time_base);
     av_seek_frame(m_fmtCtx, m_videoStreamIdx, seek_target, AVSEEK_FLAG_BACKWARD);
     avcodec_flush_buffers(m_videoCodecCtx);
     if (m_audioCodecCtx) avcodec_flush_buffers(m_audioCodecCtx);
+    
     // 重置索引
     m_videoPlayIndex = 0;
     m_audioPlayIndex = 0;
+    
+    // 重置PTS
+    m_videoPts = pts;
+    m_audioPts = pts;
+    m_currentPts = pts;
+    
     // 重新启动线程
     m_running = true;
     m_decodeThread = QThread::create([this]() { this->decodeLoop(); });
     m_decodeThread->start();
-    m_playThread = QThread::create([this]() { this->playLoop(); });
-    m_playThread->start();
+    m_videoPlayThread = QThread::create([this]() { this->videoPlayLoop(); });
+    m_videoPlayThread->start();
+    m_audioPlayThread = QThread::create([this]() { this->audioPlayLoop(); });
+    m_audioPlayThread->start();
 }
 
 // 取帧接口

AvPlayer/ffmpegvideopuller.h

@@ -57,7 +57,13 @@ public:
 
 private:
     void decodeLoop();
-    void playLoop();
+    void videoPlayLoop();
+    void audioPlayLoop();
+    
+    // 音视频同步
+    std::atomic<double> m_audioPts{0.0};
+    std::atomic<double> m_videoPts{0.0};
+    std::mutex m_syncMutex;
 
     QString m_url;
     std::atomic<bool> m_running{false};
@@ -77,7 +83,8 @@ private:
 
     // 线程
     QThread* m_decodeThread = nullptr;
-    QThread* m_playThread = nullptr;
+    QThread* m_videoPlayThread = nullptr;
+    QThread* m_audioPlayThread = nullptr;
 
     // 播放指针
     std::atomic<size_t> m_videoPlayIndex{0};