/****************************************************************************** QtAV: Multimedia framework based on Qt and FFmpeg Copyright (C) 2012-2016 Wang Bin * This file is part of QtAV This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version. This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ******************************************************************************/ #include "AudioThread.h" #include "AVThread_p.h" #include "AudioDecoder.h" #include "Packet.h" #include "AudioFormat.h" #include "AudioOutput.h" #include "AudioResampler.h" #include "AVClock.h" #include "Filter.h" #include "OutputSet.h" #include "AVCompat.h" #include #include #include "Logger.h" namespace FAV { class AudioThreadPrivate : public AVThreadPrivate { public: void init() { resample = false; last_pts = 0; } bool resample; qreal last_pts; //used when audio output is not available, to calculate the aproximate sleeping time }; AudioThread::AudioThread(QObject *parent) :AVThread(*new AudioThreadPrivate(), parent) { stopPosition = -1; } void AudioThread::applyFilters(AudioFrame &frame) { DPTR_D(AudioThread); //QMutexLocker locker(&d.mutex); //Q_UNUSED(locker); if (!d.filters.isEmpty()) { //sort filters by format. vo->defaultFormat() is the last foreach (Filter *filter, d.filters) { AudioFilter *af = static_cast(filter); if (!af->isEnabled()) continue; af->apply(d.statistics, &frame); } } } /* *TODO: * if output is null or dummy, the use duration to wait */ #define LOG_AUDIO_THREAD qInfo // qInfo //#define CHECK_AUDIO_BREAK 1 // 오디오 종료,끊기는 지점 확인 void AudioThread::run() { DPTR_D(AudioThread); //No decoder or output. No audio output is ok, just display picture if (!d.dec || !d.dec->isAvailable() || !d.outputSet) return; resetState(); Q_ASSERT(d.clock != 0); d.init(); Packet pkt; qint64 fake_duration = 0LL; qint64 fake_pts = 0LL; int sync_id = 0; #if (FORCE_BREAK_EOF) forceEnd = false; aboutToEnd = false; #endif while (!d.stop) { #if (USE_DURATION_IN_THREAD) usleep(1); // 32x speed 에서 처리되지 않음 if(durationSec > 0 && d.clock->value() > durationSec) { clock()->updateValue(durationSec); //external clock? msleep(100); break; } #endif processNextTask(); #if (FORCE_BREAK_EOF) if(forceEnd) { #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO BREAK(forceEnd) LN:" << __LINE__; #endif break; } #endif const bool is_external_clock = d.clock->clockType() == AVClock::ExternalClock; if (d.render_pts0 < 0) { // no pause when seeking if (tryPause()) { //DO NOT continue, or stepForward() will fail if (d.stop) { #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO BREAK(d.stop) LN:" << __LINE__; #endif break; //the queue is empty and may block. should setBlocking(false) wake up cond empty? } } else { if (isPaused()) { #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(isPaused()) LN:" << __LINE__; #endif continue; } } } // PAUSE 에도 발생 if (d.seek_requested) { d.seek_requested = false; // LOG_AUDIO_THREAD("request seek audio thread"); pkt = Packet(); // last decode failed and pkt is valid, reset pkt to force take the next packet if seek is requested msleep(1); } else { // d.render_pts0 < 0 means seek finished here if (d.clock->syncId() > 0) { //qInfo("audio thread wait to sync end for sync id: %d", d.clock->syncId()); #if (FORCE_BREAK_EOF) if (d.render_pts0 < 0 && sync_id > 0 && aboutToEnd == false) { #else if (d.render_pts0 < 0 && sync_id > 0) { #endif msleep(10); #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(d.clock->syncId() > 0) LN:" << __LINE__; #endif continue; } } else { sync_id = 0; } } if (!pkt.isValid()) { // PACKET INVALID 원인 확인 // qInfo() << "AUDIO PACKET INVALID LN:" << __LINE__ << "isCorrupt:" << pkt.isCorrupt << " empty:" << pkt.data.isEmpty() << " pts:" << pkt.pts << " duration:" << pkt.duration; // can't seek back if eof packet is read //qDebug("eof pkt: %d valid: %d, aqueue size: %d, abuffer: %d %.3f %d, fake_duration: %lld", pkt.isEOF(), pkt.isValid(), d.packets.size(), d.packets.bufferValue(), d.packets.bufferMax(), d.packets.isFull(), fake_duration); // If seek requested but last decode failed if (!pkt.isEOF() && (fake_duration <= 0 || !d.packets.isEmpty())) { pkt = d.packets.take(); //wait to dequeue //qInfo() << "AUDIO PTS:" << pkt.dts << " CLOCK:" << d.clock->value(); } #if (FORCE_BREAK_EOF) if (pkt.isEOF() || forceEnd) #else if (pkt.isEOF()) #endif { fake_duration = 0; //avoid endless wait qDebug("audio thread gets an eof packet. pkt.pts: %.3f, d.render_pts0:%.3f", pkt.pts, d.render_pts0); } if (!pkt.isValid()) { // check seek first if (pkt.pts >= 0) { qDebug("Invalid packet! flush audio codec context!!!!!!!! audio queue size=%d", d.packets.size()); QMutexLocker locker(&d.mutex); Q_UNUSED(locker); if (d.dec) //maybe set to null in setDecoder() { d.dec->flush(); } // render_pts0 지정 d.render_pts0 = pkt.pts; sync_id = pkt.position; // LOG_AUDIO_THREAD("audio seek: %.3f, id: %d", d.render_pts0, sync_id); pkt = Packet(); //mark invalid to take next if (fake_duration > 0) { //qInfo("fake_duration update on seek: %ul + %ul - %.3f", fake_duration, fake_pts, d.render_pts0); fake_duration = fake_duration + fake_pts - d.render_pts0*1000.0; fake_pts = d.render_pts0*1000.0; } // External 일 경우 의미 없음 // LOG_AUDIO_THREAD() << "d.clock->updateValue(d.render_pts0)"; d.clock->updateValue(d.render_pts0); d.clock->updateDelay(0); continue; } if (pkt.duration > 0) { fake_duration = pkt.duration * 1000.0; fake_pts = d.last_pts*1000.0; pkt = Packet(); //mark invalid to avoid run here in the next loop //qDebug("get fake apkt: %.3f+%ul", pkt.pts, fake_duration); #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(pkt.duration > 0) LN:" << __LINE__ << " IS EOF:" << pkt.isEOF() << "forceEnd:" << forceEnd; #endif continue; } } if (fake_duration > 0) { static const ulong kSleepMs = 20; const ulong ms = qMin(fake_duration, kSleepMs); fake_duration -= ms; fake_pts += ms; //LOG_AUDIO_THREAD("fake_wait: %ul, fake_duration: %lld, delay: %.3f", ms, fake_duration, d.clock->delay()); d.clock->updateDelay(d.clock->delay() + qreal(ms)/1000.0); msleep(ms); #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(fake_duration > 0) LN:" << __LINE__ << " clock:" << d.clock->value() << " IS EOF:" << pkt.isEOF() << "forceEnd:" << forceEnd; #endif continue; } } if (!pkt.isValid() && !pkt.isEOF()) // decode it will cause crash { #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(isNotValid+NotEOF) LN:" << __LINE__; #endif continue; } qreal dts = pkt.dts; //FIXME: pts and dts // no key frame for audio. so if pts reaches, try decode and skip render if got frame pts does not reach bool skip_render = pkt.pts >= 0 && pkt.pts < d.render_pts0; // if audio stream is too short, seeking will fail and d.render_pts0 keeps >0 // audio has no key frame, skip rendering equals to skip decoding // SEEK if (skip_render) { d.clock->updateValue(pkt.pts); // audio may be too fast than video if skip without sleep // a frame is about 20ms. sleep time must be << frame time qreal a_v = dts - d.clock->videoTime(); // qInfo("skip audio decode at %f/%f v=%f a-v=%fms", dts, d.render_pts0, d.clock->videoTime(), a_v*1000.0); if (a_v > 0) { // LOG_AUDIO_THREAD() << "a_v:" << a_v; msleep(qMin((ulong)20, ulong(a_v*1000.0))); d.clock->updateValue(a_v); } else { // audio maybe too late compared with video packet before seeking backword. so just ignore // 64 로 처리하면 영상 딜레이가 발생함..!!!!!! #if (MODEL_BBVIEWER) msleep(0); //original #else msleep(0); //wait video seek done if audio done early #endif } pkt = Packet(); //mark invalid to take next #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(skip_render) LN:" << __LINE__; #endif continue; } // const bool is_external_clock = d.clock->clockType() == AVClock::ExternalClock; if (is_external_clock && !pkt.isEOF()) { d.delay = dts - d.clock->value(); // qInfo() << "dts:" << dts << " d.delay:" << d.delay; // after seeking forward, a packet may be the old, v packet may be // the new packet, then the d.delay is very large, omit it. // TODO: 1. how to choose the value // 2. use last delay when seeking // 오차가 작을경우 if (qAbs(d.delay) < 2.0) { //qInfo() << "d.delay+:" << d.delay; // 느릴 경우 (뒤로 이동) if (d.delay < -kSyncThreshold) { //Speed up. drop frame? resample? //qInfo() << "d.delay < -kSyncThreshold"; //msleep(1); // 추가 (노이즈 방지) //qInfo("audio is late compared with external clock. skip decoding. %.3f-%.3f=%.3f", dts, d.clock->value(), d.delay); pkt = Packet(); //mark invalid to take next #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(qAbs(d.delay) < 2.0) LN:" << __LINE__; #endif continue; } // 빠를 경우 (전방이동) if (d.delay > 0) { // 아래버그 처리하니 DELAY 는 해결됨 -> 노이즈가 생김... //qInfo() << "!!>>>>W" << d.delay << dts << __FUNCTION__ << __LINE__; waitAndCheck(d.delay, dts); //-> 버그로 보임 //waitAndCheck(ulong(d.delay * 100.0), dts); //waitAndCheck(ulong(d.delay * 1000.0), dts); } } else { //when to drop off? // 오차가 클 경우 (SEEK) if (d.delay > 0) // 전방이동 { msleep(64); //qInfo() << "DELAY > 0 BIG" << d.delay; } else // 후방이동 { // DELAY 가 계속 커지면서 (-) 문제가 발생함 #if (MODEL_BBVIEWER) //qInfo() << "backward d.delay-:" << d.delay; msleep(32); #else // 128 일 경우 전방 이동시 부드럽게 시작?? 하지 않는다. msleep(64); // 추가 (노이즈 방지) -> SLEEP 하지 않으면 VideoThread 처리가 늦어짐 #endif //audio packet not cleaned up? qDebug("audio is too late compared with external clock. skip decoding. %.3f-%.3f=%.3f", dts, d.clock->value(), d.delay); pkt = Packet(); //mark invalid to take next //qInfo() << "DELAY < 0 BIG" << d.delay; #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(qAbs(d.delay) > 2.0) LN:" << __LINE__; #endif continue; } } } /* lock here to ensure decoder and ao can complete current work before they are changed * current packet maybe not supported by new decoder */ // TODO: smaller scope QMutexLocker locker(&d.mutex); Q_UNUSED(locker); AudioDecoder *dec = static_cast(d.dec); if (!dec) { pkt = Packet(); //mark invalid to take next #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(!dec) LN:" << __LINE__; #endif continue; } AudioOutput *ao = 0; // first() is not null even if list empty if (!d.outputSet->outputs().isEmpty()) { ao = static_cast(d.outputSet->outputs().first()); } //DO NOT decode and convert if ao is not available or mute! bool has_ao = ao && ao->isAvailable(); //if (!has_ao) {//do not decode? // TODO: move resampler to AudioFrame, like VideoFrame does if (has_ao && dec->resampler()) { if (dec->resampler()->speed() != ao->speed() || dec->resampler()->outAudioFormat() != ao->audioFormat()) { //resample later to ensure thread safe. TODO: test if (d.resample) { qDebug() << "ao.format " << ao->audioFormat(); qDebug() << "swr.format " << dec->resampler()->outAudioFormat(); qDebug("decoder set speed: %.2f", ao->speed()); dec->resampler()->setOutAudioFormat(ao->audioFormat()); dec->resampler()->setSpeed(ao->speed()); dec->resampler()->prepare(); d.resample = false; } else { d.resample = true; } } } else { if (dec->resampler() && dec->resampler()->speed() != d.clock->speed()) { if (d.resample) { qDebug("decoder set speed: %.2f", d.clock->speed()); dec->resampler()->setSpeed(d.clock->speed()); dec->resampler()->prepare(); d.resample = false; } else { d.resample = true; } } } if (d.stop) { qDebug("audio thread stop before decode()"); #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO BREAK(d.stop) LN:" << __LINE__; #endif break; } //qDebug("apkt: %.3f, %lld %p", pkt.pts, pkt.asAVPacket()->pts, pkt.asAVPacket()->data); if (!dec->decode(pkt)) { qWarning("Decode audio failed. undecoded: %d", dec->undecodedSize()); if (pkt.isEOF()) { qDebug("audio decode eof done"); Q_EMIT eofDecoded(); if (d.render_pts0 >= 0) { qDebug("audio seek done at eof pts: %.3f. id: %d", pkt.pts, sync_id); d.render_pts0 = -1; d.clock->syncEndOnce(sync_id); #if (PLAY_SYNC_FIX2) Q_EMIT seekFinished(qint64(pkt.pts*1000.0),qint64(d.render_pts0*1000.0)); //TODO: pts #else Q_EMIT seekFinished(qint64(pkt.pts*1000.0)); //TODO: pts #endif } if (!pkt.position) { #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO BREAK(!pkt.position) LN:" << __LINE__; #endif break; } } qreal dt = dts - d.last_pts; if (dt > 0.5 || dt < 0) { dt = 0; } if (!qFuzzyIsNull(dt)) { msleep((unsigned long)(dt*1000.0)); } pkt = Packet(); d.last_pts = d.clock->value(); //not pkt.pts! the delay is updated! #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(!dec->decode(pkt)) LN:" << __LINE__; #endif continue; } // reduce here to ensure to decode the rest data in the next loop if (!pkt.isEOF()) { pkt.skip(pkt.data.size() - dec->undecodedSize()); } #if (USE_AUDIO_FRAME) AudioFrame frame(dec->frame()); if (!frame) { #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(!frame) LN:" << __LINE__; #endif // CHECK_AUDIO_BREAK continue; //pkt data is updated after decode, no reset here } if (frame.pts() <= 0) { frame.setPTS(pkt.pts); // pkt.pts is wrong. >= real timestamp } if (d.render_pts0 >= 0.0) { // seeking d.clock->updateValue(frame.pts()); if (frame.pts() < d.render_pts0) { qDebug("skip audio rendering: %f-%f", frame.pts(), d.render_pts0); #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(frame.timestamp() < d.render_pts0) LN:" << __LINE__; #endif // CHECK_AUDIO_BREAK continue; //pkt data is updated after decode, no reset here } qDebug("audio seek finished @%.3f. id: %d", frame.pts(), sync_id); #if (PLAY_SYNC_FIX2) qreal requested = d.render_pts0; #endif // PLAY_SYNC_FIX2 d.render_pts0 = -1.0; d.clock->syncEndOnce(sync_id); #if (PLAY_SYNC_FIX2) Q_EMIT seekFinished(qint64(frame.pts()*1000.0),qint64(requested*1000.0)); #else // PLAY_SYNC_FIX2 Q_EMIT seekFinished(qint64(frame.pts()*1000.0)); #endif // PLAY_SYNC_FIX2 if (has_ao) { ao->clear(); } } if (has_ao) { applyFilters(frame); frame.setAudioResampler(dec->resampler()); //!!! // FIXME: resample ONCE is required for audio frames from ffmpeg //if (ao->audioFormat() != frame.format()) { frame = frame.to(ao->audioFormat()); //} } #if (FIXED_FPS_DURATION) bool isStopPosition = (stopPosition > 0 && (qint64)(frame.pts() * 1000000) >= stopPosition); // 후방 깨진영역 플레이 하지 않는다?? if(isStopPosition == true && playerID == 0) { #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO CONTINUE(isStopPosition) LN:" << __LINE__; #endif // CHECK_AUDIO_BREAK msleep(1); continue; } #endif // FIXED_FPS_DURATION QByteArray decoded(frame.data()); #else // AUDIO FRAME QByteArray decoded(dec->data()); #endif // AUDIO FRAME int decodedSize = decoded.size(); int decodedPos = 0; qreal delay = 0; const qreal byte_rate = frame.format().bytesPerSecond(); qreal pts = frame.pts(); //qDebug("frame samples: %d @%.3f+%lld", frame.samplesPerChannel()*frame.channelCount(), frame.timestamp(), frame.duration()/1000LL); while (decodedSize > 0) { if (d.stop) { #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO BREAK(d.stop) LN:" << __LINE__; #endif break; } const int chunk = qMin(decodedSize, has_ao ? ao->bufferSize() : 512*frame.format().bytesPerFrame());//int(max_len*byte_rate)); //AudioFormat.bytesForDuration const qreal chunk_delay = (qreal)chunk/(qreal)byte_rate; if (has_ao && ao->isOpen()) { QByteArray decodedChunk = QByteArray::fromRawData(decoded.constData() + decodedPos, chunk); //LOG_AUDIO_THREAD() << "A:" << ao->timestamp(); //ao->play(decodedChunk, pts); // qInfo() << playerID << "pts" << pts << "duration" << duration << __FUNCTION__; // 사운드가 재생이 안될 경우 duration 확인 if(pts < duration) { ao->play(decodedChunk, pts); // qInfo() << "SOUND PLAY:" << ao->play(decodedChunk, pts); } // PAUSE + PLAY 시 timestamp 가 0.1초 정도 (전,후방) 이동함 // 7.794 -> 0000 -> 7.691 //qInfo("ao.timestamp: %.3f, pts: %.3f, pktpts: %.3f", ao->timestamp(), pts, pkt.pts); //if (!is_external_clock && ao->timestamp() > 0) if (!is_external_clock && ao->timestamp() > 0) { //TODO: clear ao buffer // const qreal da = qAbs(pts - ao->timestamp()); // if (da > 1.0) { // what if frame duration is long? // } // TODO: check seek_requested(atomic bool) d.clock->updateValue(ao->timestamp()); } } else // 발생하지 않는다. { d.clock->updateDelay(delay += chunk_delay); // why need this even if we add delay? and usleep sounds weird // the advantage is if no audio device, the play speed is ok too // So is portaudio blocking the thread when playing? // TODO: avoid acummulative error. External clock? msleep((unsigned long)(chunk_delay * 1000.0)); // 누적에러 처리??? // LOG_AUDIO_THREAD() << "chunk_delay:" << chunk_delay; } decodedPos += chunk; decodedSize -= chunk; pts += chunk_delay; pkt.pts += chunk_delay; // packet not fully decoded, use new pts in the next decoding pkt.dts += chunk_delay; } if (has_ao) { emit frameDelivered(); } d.last_pts = d.clock->value(); //not pkt.pts! the delay is updated! //LOG_AUDIO_THREAD() << "d.last_pts:" << d.last_pts; } d.packets.clear(); #if (CHECK_AUDIO_BREAK) qInfo() << "AUDIO END LN:" << __LINE__; #endif qDebug("Audio thread stops running..."); //qInfo() << "AUDIO STOP!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"; //FIXED_SLEEP; //qInfo() << "EXIT THREAD 0 AUDIO" << __FUNCTION__ << __LINE__; } } //namespace FAV