JerryJeong/fmviewer3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
JerryJeong
2026-02-21 17:11:31 +09:00
commit 18b4338361
4001 changed files with 365464 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

620
project/fm_viewer/fav/AudioThread.cpp Normal file
View File

@@ -0,0 +1,620 @@
/******************************************************************************
QtAV: Multimedia framework based on Qt and FFmpeg
Copyright (C) 2012-2016 Wang Bin <wbsecg1@gmail.com>
* This file is part of QtAV
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
******************************************************************************/
#include "AudioThread.h"
#include "AVThread_p.h"
#include "AudioDecoder.h"
#include "Packet.h"
#include "AudioFormat.h"
#include "AudioOutput.h"
#include "AudioResampler.h"
#include "AVClock.h"
#include "Filter.h"
#include "OutputSet.h"
#include "AVCompat.h"
#include <QtCore/QCoreApplication>
#include <QtCore/QDateTime>
#include "Logger.h"
namespace FAV {
class AudioThreadPrivate : public AVThreadPrivate
{
public:
void init() {
resample = false;
last_pts = 0;
}
bool resample;
qreal last_pts; //used when audio output is not available, to calculate the aproximate sleeping time
};
AudioThread::AudioThread(QObject *parent)
:AVThread(*new AudioThreadPrivate(), parent)
{
stopPosition = -1;
}
void AudioThread::applyFilters(AudioFrame &frame)
{
DPTR_D(AudioThread);
//QMutexLocker locker(&d.mutex);
//Q_UNUSED(locker);
if (!d.filters.isEmpty()) {
//sort filters by format. vo->defaultFormat() is the last
foreach (Filter *filter, d.filters) {
AudioFilter *af = static_cast<AudioFilter*>(filter);
if (!af->isEnabled())
continue;
af->apply(d.statistics, &frame);
}
}
}
/*
*TODO:
* if output is null or dummy, the use duration to wait
*/
#define LOG_AUDIO_THREAD qInfo // qInfo
//#define CHECK_AUDIO_BREAK 1 // 오디오 종료,끊기는 지점 확인
void AudioThread::run()
{
DPTR_D(AudioThread);
//No decoder or output. No audio output is ok, just display picture
if (!d.dec || !d.dec->isAvailable() || !d.outputSet)
return;
resetState();
Q_ASSERT(d.clock != 0);
d.init();
Packet pkt;
qint64 fake_duration = 0LL;
qint64 fake_pts = 0LL;
int sync_id = 0;
#if (FORCE_BREAK_EOF)
forceEnd = false;
aboutToEnd = false;
#endif
while (!d.stop)
{
#if (USE_DURATION_IN_THREAD)
usleep(1); // 32x speed 에서 처리되지 않음
if(durationSec > 0 && d.clock->value() > durationSec)
{
clock()->updateValue(durationSec); //external clock?
msleep(100);
break;
}
#endif
processNextTask();
#if (FORCE_BREAK_EOF)
if(forceEnd)
{
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO BREAK(forceEnd) LN:" << __LINE__;
#endif
break;
}
#endif
const bool is_external_clock = d.clock->clockType() == AVClock::ExternalClock;
if (d.render_pts0 < 0) { // no pause when seeking
if (tryPause()) { //DO NOT continue, or stepForward() will fail
if (d.stop)
{
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO BREAK(d.stop) LN:" << __LINE__;
#endif
break; //the queue is empty and may block. should setBlocking(false) wake up cond empty?
}
} else {
if (isPaused())
{
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(isPaused()) LN:" << __LINE__;
#endif
continue;
}
}
}
// PAUSE 에도 발생
if (d.seek_requested)
{
d.seek_requested = false;
// LOG_AUDIO_THREAD("request seek audio thread");
pkt = Packet(); // last decode failed and pkt is valid, reset pkt to force take the next packet if seek is requested
msleep(1);
}
else
{
// d.render_pts0 < 0 means seek finished here
if (d.clock->syncId() > 0)
{
//qInfo("audio thread wait to sync end for sync id: %d", d.clock->syncId());
#if (FORCE_BREAK_EOF)
if (d.render_pts0 < 0 && sync_id > 0 && aboutToEnd == false) {
#else
if (d.render_pts0 < 0 && sync_id > 0) {
#endif
msleep(10);
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(d.clock->syncId() > 0) LN:" << __LINE__;
#endif
continue;
}
} else {
sync_id = 0;
}
}
if (!pkt.isValid())
{
// PACKET INVALID 원인 확인
// qInfo() << "AUDIO PACKET INVALID LN:" << __LINE__ << "isCorrupt:" << pkt.isCorrupt << " empty:" << pkt.data.isEmpty() << " pts:" << pkt.pts << " duration:" << pkt.duration;
// can't seek back if eof packet is read
//qDebug("eof pkt: %d valid: %d, aqueue size: %d, abuffer: %d %.3f %d, fake_duration: %lld", pkt.isEOF(), pkt.isValid(), d.packets.size(), d.packets.bufferValue(), d.packets.bufferMax(), d.packets.isFull(), fake_duration);
// If seek requested but last decode failed
if (!pkt.isEOF() && (fake_duration <= 0 || !d.packets.isEmpty()))
{
pkt = d.packets.take(); //wait to dequeue
//qInfo() << "AUDIO PTS:" << pkt.dts << " CLOCK:" << d.clock->value();
}
#if (FORCE_BREAK_EOF)
if (pkt.isEOF() || forceEnd)
#else
if (pkt.isEOF())
#endif
{
fake_duration = 0; //avoid endless wait
qDebug("audio thread gets an eof packet. pkt.pts: %.3f, d.render_pts0:%.3f", pkt.pts, d.render_pts0);
}
if (!pkt.isValid())
{
// check seek first
if (pkt.pts >= 0)
{
qDebug("Invalid packet! flush audio codec context!!!!!!!! audio queue size=%d", d.packets.size());
QMutexLocker locker(&d.mutex);
Q_UNUSED(locker);
if (d.dec) //maybe set to null in setDecoder()
{
d.dec->flush();
}
// render_pts0 지정
d.render_pts0 = pkt.pts;
sync_id = pkt.position;
// LOG_AUDIO_THREAD("audio seek: %.3f, id: %d", d.render_pts0, sync_id);
pkt = Packet(); //mark invalid to take next
if (fake_duration > 0)
{
//qInfo("fake_duration update on seek: %ul + %ul - %.3f", fake_duration, fake_pts, d.render_pts0);
fake_duration = fake_duration + fake_pts - d.render_pts0*1000.0;
fake_pts = d.render_pts0*1000.0;
}
// External 일 경우 의미 없음
// LOG_AUDIO_THREAD() << "d.clock->updateValue(d.render_pts0)";
d.clock->updateValue(d.render_pts0);
d.clock->updateDelay(0);
continue;
}
if (pkt.duration > 0)
{
fake_duration = pkt.duration * 1000.0;
fake_pts = d.last_pts*1000.0;
pkt = Packet(); //mark invalid to avoid run here in the next loop
//qDebug("get fake apkt: %.3f+%ul", pkt.pts, fake_duration);
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(pkt.duration > 0) LN:" << __LINE__ << " IS EOF:" << pkt.isEOF() << "forceEnd:" << forceEnd;
#endif
continue;
}
}
if (fake_duration > 0)
{
static const ulong kSleepMs = 20;
const ulong ms = qMin<qint64>(fake_duration, kSleepMs);
fake_duration -= ms;
fake_pts += ms;
//LOG_AUDIO_THREAD("fake_wait: %ul, fake_duration: %lld, delay: %.3f", ms, fake_duration, d.clock->delay());
d.clock->updateDelay(d.clock->delay() + qreal(ms)/1000.0);
msleep(ms);
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(fake_duration > 0) LN:" << __LINE__ << " clock:" << d.clock->value() << " IS EOF:" << pkt.isEOF() << "forceEnd:" << forceEnd;
#endif
continue;
}
}
if (!pkt.isValid() && !pkt.isEOF()) // decode it will cause crash
{
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(isNotValid+NotEOF) LN:" << __LINE__;
#endif
continue;
}
qreal dts = pkt.dts; //FIXME: pts and dts
// no key frame for audio. so if pts reaches, try decode and skip render if got frame pts does not reach
bool skip_render = pkt.pts >= 0 && pkt.pts < d.render_pts0; // if audio stream is too short, seeking will fail and d.render_pts0 keeps >0
// audio has no key frame, skip rendering equals to skip decoding
// SEEK
if (skip_render) {
d.clock->updateValue(pkt.pts);
// audio may be too fast than video if skip without sleep
// a frame is about 20ms. sleep time must be << frame time
qreal a_v = dts - d.clock->videoTime();
// qInfo("skip audio decode at %f/%f v=%f a-v=%fms", dts, d.render_pts0, d.clock->videoTime(), a_v*1000.0);
if (a_v > 0)
{
// LOG_AUDIO_THREAD() << "a_v:" << a_v;
msleep(qMin((ulong)20, ulong(a_v*1000.0)));
d.clock->updateValue(a_v);
} else {
// audio maybe too late compared with video packet before seeking backword. so just ignore
// 64 로 처리하면 영상 딜레이가 발생함..!!!!!!
#if (MODEL_BBVIEWER)
msleep(0); //original
#else
msleep(0); //wait video seek done if audio done early
#endif
}
pkt = Packet(); //mark invalid to take next
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(skip_render) LN:" << __LINE__;
#endif
continue;
}
// const bool is_external_clock = d.clock->clockType() == AVClock::ExternalClock;
if (is_external_clock && !pkt.isEOF()) {
d.delay = dts - d.clock->value();
// qInfo() << "dts:" << dts << " d.delay:" << d.delay;
// after seeking forward, a packet may be the old, v packet may be
// the new packet, then the d.delay is very large, omit it.
// TODO: 1. how to choose the value
// 2. use last delay when seeking
// 오차가 작을경우
if (qAbs(d.delay) < 2.0) {
//qInfo() << "d.delay+:" << d.delay;
// 느릴 경우 (뒤로 이동)
if (d.delay < -kSyncThreshold) { //Speed up. drop frame? resample?
//qInfo() << "d.delay < -kSyncThreshold";
//msleep(1); // 추가 (노이즈 방지)
//qInfo("audio is late compared with external clock. skip decoding. %.3f-%.3f=%.3f", dts, d.clock->value(), d.delay);
pkt = Packet(); //mark invalid to take next
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(qAbs(d.delay) < 2.0) LN:" << __LINE__;
#endif
continue;
}
// 빠를 경우 (전방이동)
if (d.delay > 0)
{
// 아래버그 처리하니 DELAY 는 해결됨 -> 노이즈가 생김...
//qInfo() << "!!>>>>W" << d.delay << dts << __FUNCTION__ << __LINE__;
waitAndCheck(d.delay, dts); //-> 버그로 보임
//waitAndCheck(ulong(d.delay * 100.0), dts);
//waitAndCheck(ulong(d.delay * 1000.0), dts);
}
}
else { //when to drop off?
// 오차가 클 경우 (SEEK)
if (d.delay > 0) // 전방이동
{
msleep(64);
//qInfo() << "DELAY > 0 BIG" << d.delay;
}
else // 후방이동
{
// DELAY 가 계속 커지면서 (-) 문제가 발생함
#if (MODEL_BBVIEWER)
//qInfo() << "backward d.delay-:" << d.delay;
msleep(32);
#else
// 128 일 경우 전방 이동시 부드럽게 시작?? 하지 않는다.
msleep(64); // 추가 (노이즈 방지) -> SLEEP 하지 않으면 VideoThread 처리가 늦어짐
#endif
//audio packet not cleaned up?
qDebug("audio is too late compared with external clock. skip decoding. %.3f-%.3f=%.3f", dts, d.clock->value(), d.delay);
pkt = Packet(); //mark invalid to take next
//qInfo() << "DELAY < 0 BIG" << d.delay;
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(qAbs(d.delay) > 2.0) LN:" << __LINE__;
#endif
continue;
}
}
}
/* lock here to ensure decoder and ao can complete current work before they are changed
* current packet maybe not supported by new decoder
*/
// TODO: smaller scope
QMutexLocker locker(&d.mutex);
Q_UNUSED(locker);
AudioDecoder *dec = static_cast<AudioDecoder*>(d.dec);
if (!dec) {
pkt = Packet(); //mark invalid to take next
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(!dec) LN:" << __LINE__;
#endif
continue;
}
AudioOutput *ao = 0;
// first() is not null even if list empty
if (!d.outputSet->outputs().isEmpty())
{
ao = static_cast<AudioOutput*>(d.outputSet->outputs().first());
}
//DO NOT decode and convert if ao is not available or mute!
bool has_ao = ao && ao->isAvailable();
//if (!has_ao) {//do not decode?
// TODO: move resampler to AudioFrame, like VideoFrame does
if (has_ao && dec->resampler()) {
if (dec->resampler()->speed() != ao->speed()
|| dec->resampler()->outAudioFormat() != ao->audioFormat()) {
//resample later to ensure thread safe. TODO: test
if (d.resample) {
qDebug() << "ao.format " << ao->audioFormat();
qDebug() << "swr.format " << dec->resampler()->outAudioFormat();
qDebug("decoder set speed: %.2f", ao->speed());
dec->resampler()->setOutAudioFormat(ao->audioFormat());
dec->resampler()->setSpeed(ao->speed());
dec->resampler()->prepare();
d.resample = false;
} else {
d.resample = true;
}
}
} else {
if (dec->resampler() && dec->resampler()->speed() != d.clock->speed()) {
if (d.resample) {
qDebug("decoder set speed: %.2f", d.clock->speed());
dec->resampler()->setSpeed(d.clock->speed());
dec->resampler()->prepare();
d.resample = false;
} else {
d.resample = true;
}
}
}
if (d.stop) {
qDebug("audio thread stop before decode()");
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO BREAK(d.stop) LN:" << __LINE__;
#endif
break;
}
//qDebug("apkt: %.3f, %lld %p", pkt.pts, pkt.asAVPacket()->pts, pkt.asAVPacket()->data);
if (!dec->decode(pkt)) {
qWarning("Decode audio failed. undecoded: %d", dec->undecodedSize());
if (pkt.isEOF()) {
qDebug("audio decode eof done");
Q_EMIT eofDecoded();
if (d.render_pts0 >= 0) {
qDebug("audio seek done at eof pts: %.3f. id: %d", pkt.pts, sync_id);
d.render_pts0 = -1;
d.clock->syncEndOnce(sync_id);
#if (PLAY_SYNC_FIX2)
Q_EMIT seekFinished(qint64(pkt.pts*1000.0),qint64(d.render_pts0*1000.0)); //TODO: pts
#else
Q_EMIT seekFinished(qint64(pkt.pts*1000.0)); //TODO: pts
#endif
}
if (!pkt.position)
{
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO BREAK(!pkt.position) LN:" << __LINE__;
#endif
break;
}
}
qreal dt = dts - d.last_pts;
if (dt > 0.5 || dt < 0) {
dt = 0;
}
if (!qFuzzyIsNull(dt)) {
msleep((unsigned long)(dt*1000.0));
}
pkt = Packet();
d.last_pts = d.clock->value(); //not pkt.pts! the delay is updated!
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(!dec->decode(pkt)) LN:" << __LINE__;
#endif
continue;
}
// reduce here to ensure to decode the rest data in the next loop
if (!pkt.isEOF()) {
pkt.skip(pkt.data.size() - dec->undecodedSize());
}
#if (USE_AUDIO_FRAME)
AudioFrame frame(dec->frame());
if (!frame)
{
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(!frame) LN:" << __LINE__;
#endif // CHECK_AUDIO_BREAK
continue; //pkt data is updated after decode, no reset here
}
if (frame.pts() <= 0)
{
frame.setPTS(pkt.pts); // pkt.pts is wrong. >= real timestamp
}
if (d.render_pts0 >= 0.0) { // seeking
d.clock->updateValue(frame.pts());
if (frame.pts() < d.render_pts0) {
qDebug("skip audio rendering: %f-%f", frame.pts(), d.render_pts0);
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(frame.timestamp() < d.render_pts0) LN:" << __LINE__;
#endif // CHECK_AUDIO_BREAK
continue; //pkt data is updated after decode, no reset here
}
qDebug("audio seek finished @%.3f. id: %d", frame.pts(), sync_id);
#if (PLAY_SYNC_FIX2)
qreal requested = d.render_pts0;
#endif // PLAY_SYNC_FIX2
d.render_pts0 = -1.0;
d.clock->syncEndOnce(sync_id);
#if (PLAY_SYNC_FIX2)
Q_EMIT seekFinished(qint64(frame.pts()*1000.0),qint64(requested*1000.0));
#else // PLAY_SYNC_FIX2
Q_EMIT seekFinished(qint64(frame.pts()*1000.0));
#endif // PLAY_SYNC_FIX2
if (has_ao) {
ao->clear();
}
}
if (has_ao) {
applyFilters(frame);
frame.setAudioResampler(dec->resampler()); //!!!
// FIXME: resample ONCE is required for audio frames from ffmpeg
//if (ao->audioFormat() != frame.format()) {
frame = frame.to(ao->audioFormat());
//}
}
#if (FIXED_FPS_DURATION)
bool isStopPosition = (stopPosition > 0 && (qint64)(frame.pts() * 1000000) >= stopPosition);
// 후방 깨진영역 플레이 하지 않는다??
if(isStopPosition == true && playerID == 0)
{
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO CONTINUE(isStopPosition) LN:" << __LINE__;
#endif // CHECK_AUDIO_BREAK
msleep(1);
continue;
}
#endif // FIXED_FPS_DURATION
QByteArray decoded(frame.data());
#else // AUDIO FRAME
QByteArray decoded(dec->data());
#endif // AUDIO FRAME
int decodedSize = decoded.size();
int decodedPos = 0;
qreal delay = 0;
const qreal byte_rate = frame.format().bytesPerSecond();
qreal pts = frame.pts();
//qDebug("frame samples: %d @%.3f+%lld", frame.samplesPerChannel()*frame.channelCount(), frame.timestamp(), frame.duration()/1000LL);
while (decodedSize > 0) {
if (d.stop)
{
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO BREAK(d.stop) LN:" << __LINE__;
#endif
break;
}
const int chunk = qMin(decodedSize, has_ao ? ao->bufferSize() : 512*frame.format().bytesPerFrame());//int(max_len*byte_rate));
//AudioFormat.bytesForDuration
const qreal chunk_delay = (qreal)chunk/(qreal)byte_rate;
if (has_ao && ao->isOpen()) {
QByteArray decodedChunk = QByteArray::fromRawData(decoded.constData() + decodedPos, chunk);
//LOG_AUDIO_THREAD() << "A:" << ao->timestamp();
//ao->play(decodedChunk, pts);
// qInfo() << playerID << "pts" << pts << "duration" << duration << __FUNCTION__;
// 사운드가 재생이 안될 경우 duration 확인
if(pts < duration) {
ao->play(decodedChunk, pts);
// qInfo() << "SOUND PLAY:" << ao->play(decodedChunk, pts);
}
// PAUSE + PLAY 시 timestamp 가 0.1초 정도 (전,후방) 이동함
// 7.794 -> 0000 -> 7.691
//qInfo("ao.timestamp: %.3f, pts: %.3f, pktpts: %.3f", ao->timestamp(), pts, pkt.pts);
//if (!is_external_clock && ao->timestamp() > 0)
if (!is_external_clock && ao->timestamp() > 0)
{ //TODO: clear ao buffer
// const qreal da = qAbs(pts - ao->timestamp());
// if (da > 1.0) { // what if frame duration is long?
// }
// TODO: check seek_requested(atomic bool)
d.clock->updateValue(ao->timestamp());
}
}
else // 발생하지 않는다.
{
d.clock->updateDelay(delay += chunk_delay);
// why need this even if we add delay? and usleep sounds weird
// the advantage is if no audio device, the play speed is ok too
// So is portaudio blocking the thread when playing?
// TODO: avoid acummulative error. External clock?
msleep((unsigned long)(chunk_delay * 1000.0));
// 누적에러 처리???
// LOG_AUDIO_THREAD() << "chunk_delay:" << chunk_delay;
}
decodedPos += chunk;
decodedSize -= chunk;
pts += chunk_delay;
pkt.pts += chunk_delay; // packet not fully decoded, use new pts in the next decoding
pkt.dts += chunk_delay;
}
if (has_ao)
{
emit frameDelivered();
}
d.last_pts = d.clock->value(); //not pkt.pts! the delay is updated!
//LOG_AUDIO_THREAD() << "d.last_pts:" << d.last_pts;
}
d.packets.clear();
#if (CHECK_AUDIO_BREAK)
qInfo() << "AUDIO END LN:" << __LINE__;
#endif
qDebug("Audio thread stops running...");
//qInfo() << "AUDIO STOP!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!";
//FIXED_SLEEP; //qInfo() << "EXIT THREAD 0 AUDIO" << __FUNCTION__ << __LINE__;
}
} //namespace FAV