video-pc/libs/AVPlayer2/audio_play_thread.cpp

#include "audio_play_thread.h"

#include "playercontroller.h"

#include <utility>
#include <QLoggingCategory>
Q_LOGGING_CATEGORY(playerControllerAudioPlayThread, "player.controller.AudioPlayThread")

#if !NDEBUG
#define DEBUG_PLAYFILTER 0
#define WRITE_AUDIO_FILE 0
#else
#define DEBUG_PLAYFILTER 0
#define WRITE_AUDIO_FILE 0
#endif

#if WRITE_AUDIO_FILE
#include <fstream>
#endif

AudioPlayThread::AudioPlayThread(VideoState* pState)
    : m_pOutput(nullptr)
    , m_pState(pState)
{
    qRegisterMetaType<AudioData>("AudioData");
}

AudioPlayThread::~AudioPlayThread()
{
    stop();
    stop_device();
    final_resample_param();
    
    // 确保QAudioOutput被正确释放
    if (m_pOutput) {
        delete m_pOutput;
        m_pOutput = nullptr;
    }
}

void AudioPlayThread::print_device() const
{
    QAudioDeviceInfo deviceInfo = QAudioDeviceInfo::defaultOutputDevice();
    auto deviceInfos = QAudioDeviceInfo::availableDevices(QAudio::AudioInput);
    for (const QAudioDeviceInfo& deviceInfo : std::as_const(deviceInfos))
        qDebug() << "Input device name: " << deviceInfo.deviceName();

    deviceInfos = QAudioDeviceInfo::availableDevices(QAudio::AudioOutput);
    for (const QAudioDeviceInfo& deviceInfo : std::as_const(deviceInfos))
        qDebug() << "Output device name: " << deviceInfo.deviceName();

    const auto edians = deviceInfo.supportedByteOrders();
    for (const QAudioFormat::Endian& endian : edians)
        qDebug() << "Endian: " << endian;
    const auto sampleTypes = deviceInfo.supportedSampleTypes();
    for (const QAudioFormat::SampleType& sampleType : sampleTypes)
        qDebug() << "sampleType: " << sampleType;

    const auto codecs = deviceInfo.supportedCodecs();
    for (const QString& codec : codecs)
        qDebug() << "codec: " << codec;

    const auto sampleRates = deviceInfo.supportedSampleRates();
    for (const int& sampleRate : sampleRates)
        qDebug() << "sampleRate: " << sampleRate;

    const auto ChannelCounts = deviceInfo.supportedChannelCounts();
    for (const int& channelCount : ChannelCounts)
        qDebug() << "channelCount: " << channelCount;

    const auto sampleSizes = deviceInfo.supportedSampleSizes();
    for (const int& sampleSize : sampleSizes)
        qDebug() << "sampleSize: " << sampleSize;
}

bool AudioPlayThread::init_device(int sample_rate,
                                  int channel,
                                  AVSampleFormat sample_fmt,
                                  float default_vol)
{
    QAudioDeviceInfo deviceInfo = QAudioDeviceInfo::defaultOutputDevice();

    QAudioFormat format;

    format.setSampleRate(sample_rate);
    format.setChannelCount(channel);
    format.setSampleSize(8 * av_get_bytes_per_sample(sample_fmt));
    format.setCodec("audio/pcm");
    format.setByteOrder(QAudioFormat::LittleEndian);
    format.setSampleType(QAudioFormat::SignedInt);

    // qDebug("sample size=%d\n", 8 * av_get_bytes_per_sample(sample_fmt));
    if (!deviceInfo.isFormatSupported(format)) {
        qWarning() << "Raw audio format not supported!";
        return false;
    }

    // 使用智能指针管理QAudioOutput，确保异常安全
    m_pOutput = new QAudioOutput(deviceInfo, format);
    // 注意：不设置parent，手动管理生命周期以确保正确的清理顺序

    set_device_volume(default_vol);

    m_audioDevice = m_pOutput->start();
    return true;
}

float AudioPlayThread::get_device_volume() const
{
    if (m_pOutput)
        return m_pOutput->volume();

    return 0;
}

void AudioPlayThread::set_device_volume(float volume)
{
    if (m_pOutput)
        m_pOutput->setVolume(volume);
}

void AudioPlayThread::stop_device()
{
    if (m_pOutput) {
        m_pOutput->stop();
        m_pOutput->reset();
        m_audioDevice = nullptr; // 清空设备指针，避免悬空指针
    }
}

void AudioPlayThread::play_file(const QString& file)
{
    /*play pcm file directly*/
    QFile audioFile;
    audioFile.setFileName(file);
    audioFile.open(QIODevice::ReadOnly);
    m_pOutput->start(&audioFile);
}

void AudioPlayThread::play_buf(const uint8_t* buf, int datasize)
{
    if (!m_audioDevice)
        return;

    uint8_t* data = (uint8_t*) buf;
    while (datasize > 0) {
        qint64 len = m_audioDevice->write((const char*) data, datasize);
        if (len < 0)
            break;
        if (len > 0) {
            data = data + len;
            datasize -= len;
        }
        // qDebug("play buf:reslen:%d, write len:%d", len, datasize);
    }
}

void AudioPlayThread::run()
{
    qCDebug(playerControllerAudioPlayThread) << "[AudioPlayThread] run start, m_pState:" << (void*) m_pState;
    assert(m_pState);
    VideoState* is = m_pState;
    qCDebug(playerControllerAudioPlayThread) << "[AudioPlayThread] VideoState* is:" << (void*) is
                                 << ", abort_request:" << is->abort_request;
    int audio_size;
    int loop_count = 0;
    for (;;) {
        if (isExit()) {
            qCDebug(playerControllerAudioPlayThread) << "[AudioPlayThread] m_exit set, exit.";
            break;
        }
        if (is->abort_request) {
            qCDebug(playerControllerAudioPlayThread) << "[AudioPlayThread] abort_request set, exit.";
            break;
        }
        if (is->paused) {
            qCDebug(playerControllerAudioPlayThread) << "[AudioPlayThread] paused, wait.";
            std::unique_lock<std::mutex> lock(m_mutex);
            m_cv.wait_for(lock, std::chrono::milliseconds(10), [this] { return isExit(); });
            continue;
        }
        qCDebug(playerControllerAudioPlayThread)
            << "[AudioPlayThread] call audio_decode_frame, loop:" << loop_count
            << ", sampq size:" << (is ? is->sampq.size : -1);
        audio_size = audio_decode_frame(is);
        qCDebug(playerControllerAudioPlayThread) << "[AudioPlayThread] audio_decode_frame ret:" << audio_size
                                     << ", loop:" << loop_count++;
        if (audio_size < 0) {
            qCWarning(playerControllerAudioPlayThread) << "[AudioPlayThread] audio_decode_frame failed, exit.";
            break;
        }
        if (!isnan(is->audio_clock)) {
            AVCodecContext* pAudioCodex = is->auddec.avctx;
            if (pAudioCodex) {
                int bytes_per_sec = av_samples_get_buffer_size(nullptr,
                                                               pAudioCodex->ch_layout.nb_channels,
                                                               pAudioCodex->sample_rate,
                                                               AV_SAMPLE_FMT_S16,
                                                               1);
                int64_t audio_callback_time = av_gettime_relative();
                set_clock_at(&is->audclk,
                             is->audio_clock - (double) (audio_size) / bytes_per_sec,
                             is->audio_clock_serial,
                             audio_callback_time / 1000000.0);
                sync_clock_to_slave(&is->extclk, &is->audclk);
            }
        }
    }
    qCDebug(playerControllerAudioPlayThread) << "[AudioPlayThread] run end, abort_request:" << is->abort_request
                                 << ", m_exit:" << (m_exit ? m_exit->load() : -1);
}

int AudioPlayThread::audio_decode_frame(VideoState* is)
{
    int data_size;
    Frame* af;

    do {
        while (frame_queue_nb_remaining(&is->sampq) == 0) {
            if (is->eof) {
                // break;
                return -1;
            } else {
                av_usleep(1000);
                // return -1;
            }

            if (is->abort_request || isExit())
                break;
        }

        if (is->abort_request || isExit())
            return -1;

        if (!(af = frame_queue_peek_readable(&is->sampq)))
            return -1;
        frame_queue_next(&is->sampq);
    } while (af->serial != is->audioq.serial);

    /*data_size = av_samples_get_buffer_size(nullptr, af->frame->channels,
          af->frame->nb_samples,
          AVSampleFormat(af->frame->format), 1);*/

#if USE_AVFILTER_AUDIO
    data_size = av_samples_get_buffer_size(nullptr,
                                           af->frame->ch_layout.nb_channels,
                                           af->frame->nb_samples,
                                           AV_SAMPLE_FMT_S16,
                                           1);
    uint8_t* const buffer_audio = (uint8_t*) av_malloc(data_size * sizeof(uint8_t));

    memcpy(buffer_audio, af->frame->data[0], data_size);
#else
    struct SwrContext* swrCtx = m_audioResample.swrCtx;
    data_size = av_samples_get_buffer_size(nullptr,
                                           af->frame->channels,
                                           af->frame->nb_samples,
                                           AV_SAMPLE_FMT_S16,
                                           0); // AVSampleFormat(af->frame->format)
    uint8_t* buffer_audio = (uint8_t*) av_malloc(data_size * sizeof(uint8_t));

    int ret = swr_convert(swrCtx,
                          &buffer_audio,
                          af->frame->nb_samples,
                          (const uint8_t**) (af->frame->data),
                          af->frame->nb_samples);
    if (ret < 0) {
        return 0;
    }
#endif

    if (is->muted && data_size > 0)
        memset(buffer_audio, 0, data_size); // mute

#if WRITE_AUDIO_FILE
    std::ofstream myfile;
    myfile.open("audio.pcm", std::ios::out | std::ios::app | std::ios::binary);
    if (myfile.is_open()) {
        myfile.write((char*) buffer_audio, data_size);
    }
#endif

    if (m_bSendToVisual) {
        AudioData data;
        
        // 增强的缓冲区大小检查
        if (data_size <= 0) {
            qCWarning(playerControllerAudioPlayThread) << "Invalid audio data size:" << data_size;
        } else if (data_size > BUFFER_LEN) {
            qCWarning(playerControllerAudioPlayThread) << "Audio frame too large, data_size:" << data_size
                                                       << ", buffer_len:" << BUFFER_LEN << ", will truncate";
        }

        // 安全的内存拷贝
        if (data_size > 0 && buffer_audio) {
            int len = std::min(data_size, BUFFER_LEN);
            memcpy(data.buffer, buffer_audio, len);
            data.len = len;
            
            // 安全地调用回调函数
            try {
                if (m_onDataVisualReady) {
                    m_onDataVisualReady(data);
                }
            } catch (...) {
                qCWarning(playerControllerAudioPlayThread) << "Exception in visual data callback";
            }
        }
    }

    play_buf(buffer_audio, data_size);

    av_free((void*) buffer_audio);

    /* update the audio clock with the pts */
    if (!isnan(af->pts)) {
        // is->audio_clock = af->pts + (double)af->frame->nb_samples /
        // af->frame->sample_rate;
        double frame = (double) af->frame->nb_samples / af->frame->sample_rate;
        // frame = frame * is->audio_speed;
        is->audio_clock = af->pts + frame;

#if USE_AVFILTER_AUDIO
        is->audio_clock = is->audio_clock_old
                          + (is->audio_clock - is->audio_clock_old) * is->audio_speed;
        // is->audio_clock = is->audio_clock * is->audio_speed;
#endif

#if DEBUG_PLAYFILTER
        static int pks_num = 0;
        pks_num++;

        qDebug("[%d]audio: clock=%0.3f pts=%0.3f, (nb:%d, sr:%d)frame:%0.3f\n",
               pks_num,
               is->audio_clock,
               af->pts,
               af->frame->nb_samples,
               af->frame->sample_rate,
               frame);

        // qDebug("audio: clock=%0.3f pts=%0.3f, (nb:%d, sr:%d)frame:%0.3f\n",
        // is->audio_clock, af->pts, af->frame->nb_samples, af->frame->sample_rate,
        // frame);
#endif
    } else {
        is->audio_clock = NAN;
    }
    is->audio_clock_serial = af->serial;

    if (m_onUpdatePlayTime)
        m_onUpdatePlayTime();

#if (!NDEBUG && PRINT_PACKETQUEUE_AUDIO_INFO)
    {
        static double last_clock;
        qDebug("audio: delay=%0.3f clock=%0.3f\n", is->audio_clock - last_clock, is->audio_clock);
        last_clock = is->audio_clock;
    }
#endif

    return data_size;
}

bool AudioPlayThread::init_resample_param(AVCodecContext* pAudio,
                                          AVSampleFormat sample_fmt,
                                          VideoState* is)
{
    if (pAudio) {
        int ret = -1;
        struct SwrContext* swrCtx = nullptr;
#if USE_AVFILTER_AUDIO
        if (is) {
            AVFilterContext* sink = is->out_audio_filter;
            // int sample_rate = av_buffersink_get_sample_rate(sink);
            // int nb_channels = av_buffersink_get_channels(sink);

            AVChannelLayout channel_layout;
            av_buffersink_get_ch_layout(sink, &channel_layout);
            // int64_t channel_layout = av_buffersink_get_channel_layout(sink);
            int format = av_buffersink_get_format(sink);

            ret = swr_alloc_set_opts2(&swrCtx,
                                      &pAudio->ch_layout,
                                      sample_fmt,
                                      pAudio->sample_rate,
                                      &channel_layout,
                                      (AVSampleFormat) format,
                                      pAudio->sample_rate,
                                      0,
                                      nullptr);

            /*m_audioResample.channel_layout = channel_layout;
      m_audioResample.sample_fmt = sample_fmt;
      m_audioResample.sample_rate = pAudio->sample_rate;*/
        }
#else
        ret = swr_alloc_set_opts2(&swrCtx,
                                  &pAudio->ch_layout,
                                  sample_fmt,
                                  pAudio->sample_rate,
                                  &pAudio->ch_layout,
                                  pAudio->sample_fmt,
                                  pAudio->sample_rate,
                                  0,
                                  nullptr);
#endif

        if (!(ret < 0)) {
            swr_init(swrCtx);
            m_audioResample.swrCtx = swrCtx;
            return true;
        }
    }
    return false;
}

void AudioPlayThread::final_resample_param()
{
    // 安全释放重采样上下文
    if (m_audioResample.swrCtx) {
        swr_free(&m_audioResample.swrCtx);
        m_audioResample.swrCtx = nullptr;
    }
}