app: implement ffmpeg

Allows viewing SMKs and FLCs. BMP and WAV view has also been moved to FFmpeg.

CMakeLists.txt

@@ -4,5 +4,7 @@ project(libweaver VERSION 1.0 LANGUAGES CXX)
 
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+
 add_subdirectory(lib)
 add_subdirectory(app)

app/CMakeLists.txt

@@ -3,6 +3,16 @@ find_package(Qt5)
 find_package(QT NAMES Qt6 Qt5 REQUIRED COMPONENTS Widgets Multimedia)
 find_package(Qt${QT_VERSION_MAJOR} REQUIRED COMPONENTS Widgets Multimedia)
 
+find_package(FFMPEG 3.0 REQUIRED
+  COMPONENTS
+  avutil
+  avcodec
+  avformat
+  avfilter
+  swscale
+  swresample
+)
+
 set(PROJECT_SOURCES
   siview/chunkmodel.cpp
   siview/chunkmodel.h
@@ -11,10 +21,8 @@ set(PROJECT_SOURCES
   siview/siview.cpp
   siview/siview.h
 
-  viewer/bitmappanel.cpp
+  viewer/mediapanel.cpp
-  viewer/bitmappanel.h
+  viewer/mediapanel.h
-  viewer/wavpanel.cpp
-  viewer/wavpanel.h
 
   main.cpp
   mainwindow.cpp
@@ -42,8 +50,17 @@ else()
   )
 endif()
 
-target_link_libraries(si-edit PRIVATE Qt${QT_VERSION_MAJOR}::Widgets Qt${QT_VERSION_MAJOR}::Multimedia libweaver)
+target_link_libraries(si-edit PRIVATE
-target_include_directories(si-edit PRIVATE "${CMAKE_SOURCE_DIR}/lib")
+  Qt${QT_VERSION_MAJOR}::Widgets
+  Qt${QT_VERSION_MAJOR}::Multimedia
+  FFMPEG::avutil
+  FFMPEG::avcodec
+  FFMPEG::avformat
+  FFMPEG::avfilter
+  FFMPEG::swscale
+  FFMPEG::swresample
+  libweaver)
+target_include_directories(si-edit PRIVATE "${CMAKE_SOURCE_DIR}/lib" ${FFMPEG_INCLUDE_DIRS})
 if (NOT MSVC)
   target_compile_options(si-edit PRIVATE -Werror)
 endif()

app/main.cpp

@@ -3,8 +3,42 @@
 #include <QApplication>
 #include <QCommandLineParser>
 
+void DebugHandler(QtMsgType type, const QMessageLogContext &context, const QString &msg)
+{
+  QByteArray localMsg = msg.toLocal8Bit();
+
+  const char* msg_type = "UNKNOWN";
+  switch (type) {
+  case QtDebugMsg:
+    msg_type = "DEBUG";
+    break;
+  case QtInfoMsg:
+    msg_type = "INFO";
+    break;
+  case QtWarningMsg:
+    msg_type = "WARNING";
+    break;
+  case QtCriticalMsg:
+    msg_type = "ERROR";
+    break;
+  case QtFatalMsg:
+    msg_type = "FATAL";
+    break;
+  }
+
+  fprintf(stderr, "[%s] %s (%s:%u)\n", msg_type, localMsg.constData(), context.function, context.line);
+
+#ifdef Q_OS_WINDOWS
+  // Windows still seems to buffer stderr and we want to see debug messages immediately, so here we make sure each line
+  // is flushed
+  fflush(stderr);
+#endif
+}
+
 int main(int argc, char *argv[])
 {
+  qInstallMessageHandler(DebugHandler);
+
   QApplication a(argc, argv);
 
   MainWindow w;

app/mainwindow.cpp

@@ -55,11 +55,8 @@ MainWindow::MainWindow(QWidget *parent) :
   panel_blank_ = new Panel();
   config_stack_->addWidget(panel_blank_);
 
-  panel_wav_ = new WavPanel();
+  panel_media_ = new MediaPanel();
-  config_stack_->addWidget(panel_wav_);
+  config_stack_->addWidget(panel_media_);
-
-  panel_bmp_ = new BitmapPanel();
-  config_stack_->addWidget(panel_bmp_);
 
   InitializeMenuBar();
 
@@ -70,10 +67,14 @@ MainWindow::MainWindow(QWidget *parent) :
 
 void MainWindow::OpenFilename(const QString &s)
 {
+  tree_->clearSelection();
+  SetPanel(panel_blank_, nullptr);
   model_.SetCore(nullptr);
 
   if (OpenInterleafFileInternal(this, &interleaf_, s)) {
+    //tree_->blockSignals(true);
     model_.SetCore(&interleaf_);
+//    tree_->blockSignals(false);
   }
 }
 
@@ -237,14 +238,12 @@ void MainWindow::SelectionChanged(const QModelIndex &index)
 
   if (c) {
     switch (c->filetype()) {
-    case MxOb::WAV:
-      p = panel_wav_;
-      break;
     case MxOb::STL:
-      p = panel_bmp_;
+    case MxOb::WAV:
-      break;
     case MxOb::SMK:
     case MxOb::FLC:
+      p = panel_media_;
+      break;
     case MxOb::OBJ:
       break;
     }

app/mainwindow.h

@@ -10,8 +10,7 @@
 
 #include "objectmodel.h"
 #include "panel.h"
-#include "viewer/bitmappanel.h"
+#include "viewer/mediapanel.h"
-#include "viewer/wavpanel.h"
 
 class MainWindow : public QMainWindow
 {
@@ -44,8 +43,7 @@ private:
   QGroupBox *action_grp_;
 
   Panel *panel_blank_;
-  WavPanel *panel_wav_;
+  MediaPanel *panel_media_;
-  BitmapPanel *panel_bmp_;
 
   ObjectModel model_;
   si::Interleaf interleaf_;

app/viewer/bitmappanel.cpp

@@ -1,39 +0,0 @@
-#include "bitmappanel.h"
-
-#include <object.h>
-#include <QBuffer>
-#include <QGroupBox>
-
-BitmapPanel::BitmapPanel(QWidget *parent)
-{
-  int row = 0;
-
-  auto preview_grp = new QGroupBox(tr("Bitmap"));
-  layout()->addWidget(preview_grp, row, 0, 1, 2);
-
-  auto preview_layout = new QVBoxLayout(preview_grp);
-
-  img_lbl_ = new QLabel();
-  img_lbl_->setAlignment(Qt::AlignHCenter);
-  preview_layout->addWidget(img_lbl_);
-
-  desc_lbl_ = new QLabel();
-  desc_lbl_->setAlignment(Qt::AlignHCenter);
-  preview_layout->addWidget(desc_lbl_);
-
-  FinishLayout();
-}
-
-void BitmapPanel::OnOpeningData(void *data)
-{
-  si::Object *o = static_cast<si::Object*>(data);
-  si::bytearray d = o->ExtractToMemory();
-  QByteArray b(d.data(), d.size());
-  QBuffer read_buf(&b);
-  QImage img;
-  img.load(&read_buf, "BMP");
-
-  img_lbl_->setPixmap(QPixmap::fromImage(img));
-
-  desc_lbl_->setText(tr("%1x%2").arg(QString::number(img.width()), QString::number(img.height())));
-}

app/viewer/bitmappanel.h

@@ -1,24 +0,0 @@
-#ifndef BITMAPPANEL_H
-#define BITMAPPANEL_H
-
-#include <panel.h>
-#include <QLabel>
-
-class BitmapPanel : public Panel
-{
-  Q_OBJECT
-public:
-  BitmapPanel(QWidget *parent = nullptr);
-
-protected:
-  virtual void OnOpeningData(void *data) override;
-  //virtual void OnClosingData(void *data) override;
-
-private:
-  QLabel *img_lbl_;
-
-  QLabel *desc_lbl_;
-
-};
-
-#endif // BITMAPPANEL_H

app/viewer/mediapanel.cpp

@@ -0,0 +1,580 @@
+#include "mediapanel.h"
+
+#include <object.h>
+#include <QDateTime>
+#include <QDebug>
+#include <QGroupBox>
+#include <QMouseEvent>
+
+MediaPanel::MediaPanel(QWidget *parent) :
+  Panel(parent),
+  m_FmtCtx(nullptr),
+  m_Packet(nullptr),
+  m_VideoCodecCtx(nullptr),
+  m_VideoStream(nullptr),
+  m_SwsFrame(nullptr),
+  m_AudioCodecCtx(nullptr),
+  m_AudioStream(nullptr),
+  m_SwsCtx(nullptr),
+  m_SwrCtx(nullptr),
+  m_IoCtx(nullptr),
+  m_AudioOutput(nullptr),
+  m_SliderPressed(false)
+{
+  int row = 0;
+
+  auto wav_group = new QGroupBox(tr("Playback"));
+  layout()->addWidget(wav_group, row, 0, 1, 2);
+
+  auto preview_layout = new QVBoxLayout(wav_group);
+
+  m_ImgViewer = new QLabel();
+  m_ImgViewer->setAlignment(Qt::AlignCenter);
+  preview_layout->addWidget(m_ImgViewer);
+
+  auto wav_layout = new QHBoxLayout();
+  preview_layout->addLayout(wav_layout);
+
+  m_PlayheadSlider = new ClickableSlider(Qt::Horizontal);
+  m_PlayheadSlider->setMinimum(0);
+  m_PlayheadSlider->setMaximum(100000);
+  connect(m_PlayheadSlider, &QSlider::sliderPressed, this, &MediaPanel::SliderPressed);
+  connect(m_PlayheadSlider, &QSlider::sliderMoved, this, &MediaPanel::SliderMoved);
+  connect(m_PlayheadSlider, &QSlider::sliderReleased, this, &MediaPanel::SliderReleased);
+  wav_layout->addWidget(m_PlayheadSlider);
+
+  m_PlayBtn = new QPushButton(tr("Play"));
+  m_PlayBtn->setCheckable(true);
+  connect(m_PlayBtn, &QPushButton::clicked, this, &MediaPanel::Play);
+  wav_layout->addWidget(m_PlayBtn);
+
+  FinishLayout();
+
+  m_PlaybackTimer = new QTimer(this);
+  m_PlaybackTimer->setInterval(10);
+  connect(m_PlaybackTimer, &QTimer::timeout, this, &MediaPanel::TimerUpdate);
+
+  m_AudioNotifyDevice = new MediaAudioDevice(this);
+}
+
+MediaPanel::~MediaPanel()
+{
+  Close();
+}
+
+qint64 MediaPanel::ReadAudio(char *data, qint64 maxlen)
+{
+  if (m_AudioFlushed) {
+    return 0;
+  }
+
+  while (!m_AudioFlushed && m_AudioBuffer.size() < maxlen) {
+    int ret = GetNextFrame(m_AudioCodecCtx, m_AudioStream->index, m_AudioFrame);
+    if (ret >= 0 || ret == AVERROR_EOF) {
+      const uint8_t **in_data;
+      int in_nb_samples;
+      if (ret == AVERROR_EOF) {
+        in_data = nullptr;
+        in_nb_samples = 0;
+        m_AudioFlushed = true;
+      } else {
+        in_data = const_cast<const uint8_t**>(m_AudioFrame->data);
+        in_nb_samples = m_AudioFrame->nb_samples;
+      }
+
+      int dst_nb_samples = av_rescale_rnd(swr_get_delay(m_SwrCtx, m_AudioStream->codecpar->sample_rate) + in_nb_samples,
+                                          m_AudioOutput->format().sampleRate(), m_AudioStream->codecpar->sample_rate, AV_ROUND_UP);
+      int data_size = dst_nb_samples * av_get_bytes_per_sample(m_AudioOutputSampleFmt) * m_AudioOutput->format().channelCount();
+
+      int old_sz = m_AudioBuffer.size();
+      m_AudioBuffer.resize(old_sz + data_size);
+
+      uint8_t *out = reinterpret_cast<uint8_t*>(m_AudioBuffer.data() + old_sz);
+      int converted = swr_convert(m_SwrCtx, &out, dst_nb_samples, in_data, in_nb_samples);
+
+      data_size = converted * av_get_bytes_per_sample(m_AudioOutputSampleFmt) * m_AudioOutput->format().channelCount();
+
+      if (m_AudioBuffer.size() != old_sz + data_size) {
+        m_AudioBuffer.resize(old_sz + data_size);
+      }
+    } else {
+      break;
+    }
+  }
+
+  if (!m_AudioBuffer.isEmpty()) {
+    qint64 copy_len = std::min(maxlen, qint64(m_AudioBuffer.size()));
+    memcpy(data, m_AudioBuffer.data(), copy_len);
+    m_AudioBuffer = m_AudioBuffer.mid(copy_len);
+    return copy_len;
+  }
+
+  return 0;
+}
+
+int ReadData(void *opaque, uint8_t *buf, int buf_sz)
+{
+  si::MemoryBuffer *m = static_cast<si::MemoryBuffer *>(opaque);
+
+  int s = m->ReadData(reinterpret_cast<char*>(buf), buf_sz);
+  if (s == 0) {
+    if (m->pos() == m->size()) {
+      s = AVERROR_EOF;
+    }
+  }
+
+  return s;
+}
+
+int64_t SeekData(void *opaque, int64_t offset, int whence)
+{
+  si::MemoryBuffer *m = static_cast<si::MemoryBuffer *>(opaque);
+
+  if (whence == AVSEEK_SIZE) {
+    return m->size();
+  }
+
+  m->seek(offset);
+
+  return m->pos();
+}
+
+void MediaPanel::OnOpeningData(void *data)
+{
+  si::Object *o = static_cast<si::Object*>(data);
+
+  m_Data = o->ExtractToMemory();
+
+  static const size_t buf_sz = 4096;
+
+  m_IoCtx = avio_alloc_context(
+        (unsigned char *) av_malloc(buf_sz),
+        buf_sz,
+        0,
+        &m_Data,
+        ReadData,
+        nullptr,
+        SeekData
+      );
+
+  m_FmtCtx = avformat_alloc_context();
+  m_FmtCtx->pb = m_IoCtx;
+  m_FmtCtx->flags |= AVFMT_FLAG_CUSTOM_IO;
+
+  if (avformat_open_input(&m_FmtCtx, "", nullptr, nullptr) < 0) {
+    qCritical() << "Failed to open format context";
+    Close();
+    return;
+  }
+
+  if (avformat_find_stream_info(m_FmtCtx, nullptr) < 0) {
+    qCritical() << "Failed to find stream info";
+    Close();
+    return;
+  }
+
+  m_Packet = av_packet_alloc();
+  m_SwsFrame = av_frame_alloc();
+  m_AudioFrame = av_frame_alloc();
+
+  for (unsigned int i=0; i<m_FmtCtx->nb_streams; i++) {
+    AVStream *s = m_FmtCtx->streams[i];
+
+    const AVCodec *decoder = avcodec_find_decoder(s->codecpar->codec_id);
+
+    if (decoder) {
+      if (!m_VideoCodecCtx && s->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) {
+        m_VideoStream = s;
+
+        m_VideoCodecCtx = avcodec_alloc_context3(decoder);
+        avcodec_parameters_to_context(m_VideoCodecCtx, s->codecpar);
+        avcodec_open2(m_VideoCodecCtx, decoder, nullptr);
+
+        const AVPixelFormat dest = AV_PIX_FMT_RGBA;
+        m_SwsCtx = sws_getContext(s->codecpar->width,
+                                  s->codecpar->height,
+                                  static_cast<AVPixelFormat>(s->codecpar->format),
+                                  s->codecpar->width,
+                                  s->codecpar->height,
+                                  dest,
+                                  0,
+                                  nullptr,
+                                  nullptr,
+                                  nullptr);
+
+        m_SwsFrame = av_frame_alloc();
+        m_SwsFrame->width = s->codecpar->width;
+        m_SwsFrame->height = s->codecpar->height;
+        m_SwsFrame->format = dest;
+        av_frame_get_buffer(m_SwsFrame, 0);
+      }
+
+      if (!m_AudioCodecCtx && s->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
+        m_AudioStream = s;
+
+        m_AudioCodecCtx = avcodec_alloc_context3(decoder);
+        avcodec_parameters_to_context(m_AudioCodecCtx, s->codecpar);
+        avcodec_open2(m_AudioCodecCtx, decoder, nullptr);
+      }
+    }
+  }
+
+  if (m_VideoCodecCtx) {
+    VideoUpdate(0);
+  }
+}
+
+void MediaPanel::OnClosingData(void *data)
+{
+  Close();
+}
+
+void MediaPanel::Close()
+{
+  Play(false);
+
+  if (m_VideoCodecCtx) {
+    avcodec_free_context(&m_VideoCodecCtx);
+  }
+
+  if (m_AudioCodecCtx) {
+    avcodec_free_context(&m_AudioCodecCtx);
+  }
+
+  if (m_SwsCtx) {
+    sws_freeContext(m_SwsCtx);
+    m_SwsCtx = nullptr;
+  }
+
+  if (m_SwrCtx) {
+    swr_free(&m_SwrCtx);
+  }
+
+  if (m_Packet) {
+    av_packet_free(&m_Packet);
+  }
+
+  if (m_SwsFrame) {
+    av_frame_free(&m_SwsFrame);
+  }
+
+  if (m_AudioFrame) {
+    av_frame_free(&m_AudioFrame);
+  }
+
+  ClearQueue();
+
+  if (m_FmtCtx) {
+    avformat_free_context(m_FmtCtx);
+    m_FmtCtx = nullptr;
+  }
+
+  if (m_IoCtx) {
+    avio_context_free(&m_IoCtx);
+    m_IoCtx = nullptr;
+  }
+
+  m_VideoStream = nullptr;
+  m_AudioStream = nullptr;
+
+  m_Data.Close();
+
+  m_PlayheadSlider->setValue(0);
+
+  m_ImgViewer->setPixmap(QPixmap());
+}
+
+void MediaPanel::VideoUpdate(float t)
+{
+  double flipped = av_q2d(av_inv_q(m_VideoStream->time_base));
+  int64_t ts = std::floor(t * flipped);
+  //int64_t second = std::ceil(flipped);
+
+  AVFrame *using_frame = nullptr;
+  for (auto it=m_FrameQueue.begin(); it!=m_FrameQueue.end(); it++) {
+    auto next = it;
+    next++;
+
+    if ((*it)->pts == ts
+        || (next != m_FrameQueue.end() && (*next)->pts > ts)) {
+      using_frame = *it;
+      break;
+    }
+  }
+
+  if (!using_frame) {
+    // Determine if the queue will eventually get this frame
+    if (m_FrameQueue.empty()
+        //|| ts > m_FrameQueue.back()->pts + second
+        || ts < m_FrameQueue.front()->pts) {
+      ClearQueue();
+      av_seek_frame(m_FmtCtx, m_VideoStream->index, ts, AVSEEK_FLAG_BACKWARD);
+    }
+
+    while (m_FrameQueue.empty() || m_FrameQueue.back()->pts < ts) {
+      AVFrame *f = av_frame_alloc();
+      int ret = GetNextFrame(m_VideoCodecCtx, m_VideoStream->index, f);
+      if (ret < 0) {
+        av_frame_free(&f);
+
+        if (ret == AVERROR_EOF) {
+          Play(false);
+        }
+        break;
+      } else {
+        AVFrame *previous = nullptr;
+        if (!m_FrameQueue.empty()) {
+          previous = m_FrameQueue.back();
+        }
+
+        m_FrameQueue.push_back(f);
+
+        if (previous && f->pts > ts) {
+          using_frame = previous;
+          break;
+        } else if (f->pts == ts) {
+          using_frame = f;
+          break;
+        }
+      }
+
+    }
+  }
+
+  if (using_frame) {
+    if (using_frame->pts != m_SwsFrame->pts) {
+      m_SwsFrame->pts = using_frame->pts;
+
+      sws_scale(m_SwsCtx, using_frame->data, using_frame->linesize, 0, using_frame->height,
+                m_SwsFrame->data, m_SwsFrame->linesize);
+
+      QImage img(m_SwsFrame->data[0], m_SwsFrame->width, m_SwsFrame->height, m_SwsFrame->linesize[0], QImage::Format_RGBA8888);
+      m_ImgViewer->setPixmap(QPixmap::fromImage(img));
+    }
+  }
+}
+
+void MediaPanel::AudioSeek(float t)
+{
+  double flipped = av_q2d(av_inv_q(m_AudioStream->time_base));
+  int64_t ts = std::floor(t * flipped);
+
+  av_seek_frame(m_FmtCtx, m_AudioStream->index, ts, AVSEEK_FLAG_BACKWARD);
+}
+
+int MediaPanel::GetNextFrame(AVCodecContext *cctx, unsigned int stream, AVFrame *frame)
+{
+  int ret;
+  av_frame_unref(frame);
+  while ((ret = avcodec_receive_frame(cctx, frame)) == AVERROR(EAGAIN)) {
+    av_packet_unref(m_Packet);
+    ret = av_read_frame(m_FmtCtx, m_Packet);
+    if (ret < 0) {
+      return ret;
+    }
+
+    if (m_Packet->stream_index == stream) {
+      ret = avcodec_send_packet(cctx, m_Packet);
+      if (ret < 0) {
+        return ret;
+      }
+    }
+  }
+
+  return ret;
+}
+
+void MediaPanel::ClearQueue()
+{
+  while (!m_FrameQueue.empty()) {
+    av_frame_free(&m_FrameQueue.front());
+    m_FrameQueue.pop_front();
+  }
+}
+
+void MediaPanel::StartAudioPlayback()
+{
+  auto output_dev = QAudioDeviceInfo::defaultOutputDevice();
+  auto fmt = output_dev.preferredFormat();
+
+  AVSampleFormat smp_fmt = AV_SAMPLE_FMT_S16;
+  switch (fmt.sampleType()) {
+  case QAudioFormat::Unknown:
+    break;
+  case QAudioFormat::SignedInt:
+    switch (fmt.sampleSize()) {
+    case 16:
+      smp_fmt = AV_SAMPLE_FMT_S16;
+      break;
+    case 32:
+      smp_fmt = AV_SAMPLE_FMT_S32;
+      break;
+    case 64:
+      smp_fmt = AV_SAMPLE_FMT_S64;
+      break;
+    }
+    break;
+  case QAudioFormat::UnSignedInt:
+    switch (fmt.sampleSize()) {
+    case 8:
+      smp_fmt = AV_SAMPLE_FMT_U8;
+      break;
+    }
+    break;
+  case QAudioFormat::Float:
+    switch (fmt.sampleSize()) {
+    case 32:
+      smp_fmt = AV_SAMPLE_FMT_FLT;
+      break;
+    case 64:
+      smp_fmt = AV_SAMPLE_FMT_DBL;
+      break;
+    }
+    break;
+  }
+  m_AudioOutputSampleFmt = smp_fmt;
+
+  m_SwrCtx = swr_alloc_set_opts(nullptr,
+                                av_get_default_channel_layout(fmt.channelCount()),
+                                smp_fmt,
+                                fmt.sampleRate(),
+                                av_get_default_channel_layout(m_AudioStream->codecpar->channels),
+                                static_cast<AVSampleFormat>(m_AudioStream->codecpar->format),
+                                m_AudioStream->codecpar->sample_rate,
+                                0, nullptr);
+  if (!m_SwrCtx) {
+    qCritical() << "Failed to alloc swr ctx";
+  } else {
+    if (swr_init(m_SwrCtx) < 0) {
+      qCritical() << "Failed to init swr ctx";
+    } else {
+      if (m_AudioFlushed) {
+        AudioSeek(0);
+        m_AudioFlushed = false;
+      }
+      m_AudioBuffer.clear();
+
+      m_AudioOutput = new QAudioOutput(output_dev, fmt, this);
+      m_AudioNotifyDevice->open(QIODevice::ReadOnly);
+      connect(m_AudioOutput, &QAudioOutput::stateChanged, this, &MediaPanel::AudioStateChanged);
+      m_AudioOutput->start(m_AudioNotifyDevice);
+    }
+  }
+}
+
+float MediaPanel::SliderValueToFloatSeconds(int i, int max, AVStream *s)
+{
+  float percent = float(i) / float(max);
+  float duration = float(s->time_base.num) * float(s->duration) / float(s->time_base.den);
+  return percent * duration;
+}
+
+void MediaPanel::Play(bool e)
+{
+  if (m_AudioOutput) {
+    m_AudioOutput->stop();
+    delete m_AudioOutput;
+    m_AudioOutput = nullptr;
+
+    m_AudioNotifyDevice->close();
+  }
+
+  if (e) {
+    if (m_VideoStream) {
+      m_PlaybackOffset = SliderValueToFloatSeconds(m_PlayheadSlider->value(), m_PlayheadSlider->maximum(), m_VideoStream);
+    } else {
+      m_PlaybackOffset = 0;
+    }
+
+    if (m_AudioStream) {
+      StartAudioPlayback();
+    }
+
+    m_PlaybackStart = QDateTime::currentMSecsSinceEpoch();
+    m_PlaybackTimer->start();
+  } else {
+    m_PlaybackTimer->stop();
+  }
+  m_PlayBtn->setChecked(e);
+}
+
+void MediaPanel::TimerUpdate()
+{
+  float now = float(QDateTime::currentMSecsSinceEpoch() - m_PlaybackStart) * 0.001f;
+  if (m_VideoStream) {
+    VideoUpdate(now);
+
+    if (!m_SliderPressed && m_SwsFrame->pts != AV_NOPTS_VALUE) {
+      float percent = float(m_SwsFrame->pts) / float(m_VideoStream->duration);
+      m_PlayheadSlider->setValue(percent * m_PlayheadSlider->maximum());
+    }
+  } else if (m_AudioStream) {
+    if (!m_SliderPressed && m_AudioFrame->pts != AV_NOPTS_VALUE) {
+      float percent = float(m_AudioFrame->pts) / float(m_AudioStream->duration);
+      m_PlayheadSlider->setValue(percent * m_PlayheadSlider->maximum());
+    }
+  }
+}
+
+void MediaPanel::SliderPressed()
+{
+  m_SliderPressed = true;
+}
+
+void MediaPanel::SliderMoved(int i)
+{
+  if (m_VideoStream) {
+    VideoUpdate(SliderValueToFloatSeconds(i, m_PlayheadSlider->maximum(), m_VideoStream));
+  }
+  if (m_AudioStream) {
+    AudioSeek(SliderValueToFloatSeconds(i, m_PlayheadSlider->maximum(), m_AudioStream));
+  }
+}
+
+void MediaPanel::SliderReleased()
+{
+  m_SliderPressed = false;
+}
+
+void MediaPanel::AudioStateChanged(QAudio::State state)
+{
+  if (state == QAudio::IdleState) {
+    Play(false);
+    m_PlayheadSlider->setValue(m_PlayheadSlider->maximum());
+  }
+}
+
+MediaAudioDevice::MediaAudioDevice(MediaPanel *o) :
+  QIODevice(o)
+{
+  m_MediaPanel = o;
+}
+
+qint64 MediaAudioDevice::readData(char *data, qint64 maxSize)
+{
+  return m_MediaPanel->ReadAudio(data, maxSize);
+}
+
+qint64 MediaAudioDevice::writeData(const char *data, qint64 maxSize)
+{
+  return -1;
+}
+
+ClickableSlider::ClickableSlider(Qt::Orientation orientation, QWidget *parent) :
+  QSlider(orientation, parent)
+{
+}
+
+ClickableSlider::ClickableSlider(QWidget *parent) :
+  QSlider(parent)
+{
+}
+
+void ClickableSlider::mousePressEvent(QMouseEvent *e)
+{
+  int v = double(e->pos().x()) / double(width()) * this->maximum();
+  setValue(v);
+  emit sliderMoved(v);
+
+  QSlider::mousePressEvent(e);
+}

app/viewer/mediapanel.h

@@ -0,0 +1,120 @@
+#ifndef MEDIAPANEL_H
+#define MEDIAPANEL_H
+
+extern "C" {
+#include <libavformat/avformat.h>
+#include <libavcodec/avcodec.h>
+#include <libswscale/swscale.h>
+#include <libswresample/swresample.h>
+}
+
+#include <file.h>
+#include <QAudioOutput>
+#include <QLabel>
+#include <QPushButton>
+#include <QSlider>
+#include <QTimer>
+
+#include "panel.h"
+
+class MediaPanel : public Panel
+{
+  Q_OBJECT
+public:
+  MediaPanel(QWidget *parent = nullptr);
+  virtual ~MediaPanel() override;
+
+  qint64 ReadAudio(char *data, qint64 maxlen);
+
+protected:
+  virtual void OnOpeningData(void *data) override;
+  virtual void OnClosingData(void *data) override;
+
+private:
+  void Close();
+
+  void VideoUpdate(float t);
+  void AudioSeek(float t);
+
+  int GetNextFrame(AVCodecContext *cctx, unsigned int stream, AVFrame *frame);
+
+  void ClearQueue();
+
+  void StartAudioPlayback();
+
+  static float SliderValueToFloatSeconds(int i, int max, AVStream *s);
+
+  AVFormatContext *m_FmtCtx;
+  AVPacket *m_Packet;
+  std::list<AVFrame*> m_FrameQueue;
+
+  AVCodecContext *m_VideoCodecCtx;
+  AVStream *m_VideoStream;
+  AVFrame *m_SwsFrame;
+  AVFrame *m_AudioFrame;
+
+  AVCodecContext *m_AudioCodecCtx;
+  AVStream *m_AudioStream;
+
+  SwsContext *m_SwsCtx;
+  SwrContext *m_SwrCtx;
+
+  AVIOContext *m_IoCtx;
+
+  si::MemoryBuffer m_Data;
+
+  QLabel *m_ImgViewer;
+
+  QAudioOutput *m_AudioOutput;
+  QIODevice *m_AudioNotifyDevice;
+  QByteArray m_AudioBuffer;
+  AVSampleFormat m_AudioOutputSampleFmt;
+  QSlider *m_PlayheadSlider;
+  QPushButton *m_PlayBtn;
+  QTimer *m_PlaybackTimer;
+  qint64 m_PlaybackStart;
+  float m_PlaybackOffset;
+  bool m_AudioFlushed;
+  bool m_SliderPressed;
+
+private slots:
+  void Play(bool e);
+
+  void TimerUpdate();
+
+  void SliderPressed();
+  void SliderMoved(int i);
+  void SliderReleased();
+
+  void AudioStateChanged(QAudio::State state);
+
+};
+
+class MediaAudioDevice : public QIODevice
+{
+  Q_OBJECT
+public:
+  MediaAudioDevice(MediaPanel *o = nullptr);
+
+protected:
+  virtual qint64 readData(char *data, qint64 maxSize) override;
+  virtual qint64 writeData(const char *data, qint64 maxSize) override;
+
+private:
+  MediaPanel *m_MediaPanel;
+
+};
+
+class ClickableSlider : public QSlider
+{
+  Q_OBJECT
+public:
+  ClickableSlider(Qt::Orientation orientation, QWidget *parent = nullptr);
+  ClickableSlider(QWidget *parent = nullptr);
+
+protected:
+  virtual void mousePressEvent(QMouseEvent *e) override;
+
+};
+
+#endif // MEDIAPANEL_H

app/viewer/wavpanel.cpp

@@ -1,117 +0,0 @@
-#include "wavpanel.h"
-
-#include <iostream>
-#include <object.h>
-#include <QGroupBox>
-#include <QHBoxLayout>
-
-WavPanel::WavPanel(QWidget *parent) :
-  Panel(parent),
-  audio_out_(nullptr)
-{
-  int row = 0;
-
-  auto wav_group = new QGroupBox(tr("Playback"));
-  layout()->addWidget(wav_group, row, 0, 1, 2);
-
-  auto wav_layout = new QHBoxLayout(wav_group);
-
-  playhead_slider_ = new QSlider(Qt::Horizontal);
-  playhead_slider_->setMinimum(0);
-  connect(playhead_slider_, &QSlider::valueChanged, this, &WavPanel::SliderMoved);
-  wav_layout->addWidget(playhead_slider_);
-
-  play_btn_ = new QPushButton(tr("Play"));
-  play_btn_->setCheckable(true);
-  connect(play_btn_, &QPushButton::clicked, this, &WavPanel::Play);
-  wav_layout->addWidget(play_btn_);
-
-  FinishLayout();
-
-  buffer_.setBuffer(&array_);
-
-  playback_timer_ = new QTimer(this);
-  playback_timer_->setInterval(100);
-  connect(playback_timer_, &QTimer::timeout, this, &WavPanel::TimerUpdate);
-}
-
-void WavPanel::OnOpeningData(void *data)
-{
-  si::Object *o = static_cast<si::Object*>(data);
-
-  // Find fmt and data
-  header_ = *o->GetFileHeader().cast<si::WAVFmt>();
-  playhead_slider_->setMaximum(o->GetFileBodySize()/GetSampleSize());
-}
-
-void WavPanel::OnClosingData(void *data)
-{
-  Play(false);
-  playhead_slider_->setValue(0);
-}
-
-int WavPanel::GetSampleSize() const
-{
-  return (header_.BitsPerSample/8) * header_.Channels;
-}
-
-void WavPanel::Play(bool e)
-{
-  if (audio_out_) {
-    audio_out_->stop();
-    delete audio_out_;
-    audio_out_ = nullptr;
-  }
-  buffer_.close();
-  array_.clear();
-  playback_timer_->stop();
-
-  if (e) {
-    si::Object *o = static_cast<si::Object*>(GetData());
-    si::bytearray pcm = o->GetFileBody();
-    array_ = QByteArray(pcm.data(), pcm.size());
-    buffer_.open(QBuffer::ReadOnly);
-
-    size_t start = 0;
-
-    if (playhead_slider_->value() < playhead_slider_->maximum()) {
-      start += playhead_slider_->value() * GetSampleSize();
-    }
-
-    buffer_.seek(start);
-
-    QAudioFormat audio_fmt;
-    audio_fmt.setSampleRate(header_.SampleRate);
-    audio_fmt.setChannelCount(header_.Channels);
-    audio_fmt.setSampleSize(header_.BitsPerSample);
-    audio_fmt.setByteOrder(QAudioFormat::LittleEndian);
-    audio_fmt.setCodec(QStringLiteral("audio/pcm"));
-    audio_fmt.setSampleType(QAudioFormat::SignedInt);
-
-    audio_out_ = new QAudioOutput(audio_fmt, this);
-    connect(audio_out_, &QAudioOutput::stateChanged, this, &WavPanel::OutputChanged);
-    audio_out_->start(&buffer_);
-
-    playback_timer_->start();
-  }
-}
-
-void WavPanel::TimerUpdate()
-{
-  playhead_slider_->setValue(buffer_.pos() / GetSampleSize());
-}
-
-void WavPanel::OutputChanged(QAudio::State state)
-{
-  if (state != QAudio::ActiveState) {
-    Play(false);
-    play_btn_->setChecked(false);
-  }
-}
-
-void WavPanel::SliderMoved(int i)
-{
-  if (buffer_.isOpen()) {
-    buffer_.seek(i * GetSampleSize());
-  }
-}

app/viewer/wavpanel.h

@@ -1,48 +0,0 @@
-#ifndef WAVPANEL_H
-#define WAVPANEL_H
-
-#include <othertypes.h>
-#include <QAudioOutput>
-#include <QBuffer>
-#include <QByteArray>
-#include <QPushButton>
-#include <QSlider>
-#include <QTimer>
-
-#include "panel.h"
-
-class WavPanel : public Panel
-{
-  Q_OBJECT
-public:
-  WavPanel(QWidget *parent = nullptr);
-
-protected:
-  virtual void OnOpeningData(void *data) override;
-  virtual void OnClosingData(void *data) override;
-
-private:
-  int GetSampleSize() const;
-
-  QSlider *playhead_slider_;
-  QPushButton *play_btn_;
-
-  QAudioOutput *audio_out_;
-  QBuffer buffer_;
-  QByteArray array_;
-  QTimer *playback_timer_;
-  si::WAVFmt header_;
-  QByteArray play_buffer_;
-
-private slots:
-  void Play(bool e);
-
-  void TimerUpdate();
-
-  void OutputChanged(QAudio::State state);
-
-  void SliderMoved(int i);
-
-};
-
-#endif // WAVPANEL_H

cmake/FindFFMPEG.cmake

@@ -0,0 +1,193 @@
+#[==[
+Provides the following variables:
+
+  * `FFMPEG_INCLUDE_DIRS`: Include directories necessary to use FFMPEG.
+  * `FFMPEG_LIBRARIES`: Libraries necessary to use FFMPEG. Note that this only
+    includes libraries for the components requested.
+  * `FFMPEG_VERSION`: The version of FFMPEG found.
+
+The following components are supported:
+
+  * `avcodec`
+  * `avdevice`
+  * `avfilter`
+  * `avformat`
+  * `avresample`
+  * `avutil`
+  * `swresample`
+  * `swscale`
+
+For each component, the following are provided:
+
+  * `FFMPEG_<component>_FOUND`: Libraries for the component.
+  * `FFMPEG_<component>_INCLUDE_DIRS`: Include directories for
+    the component.
+  * `FFMPEG_<component>_LIBRARIES`: Libraries for the component.
+  * `FFMPEG::<component>`: A target to use with `target_link_libraries`.
+
+Note that only components requested with `COMPONENTS` or `OPTIONAL_COMPONENTS`
+are guaranteed to set these variables or provide targets.
+#]==]
+
+function (_ffmpeg_find component headername)
+  find_path("FFMPEG_${component}_INCLUDE_DIR"
+    NAMES
+      "lib${component}/${headername}"
+    PATHS
+      "${FFMPEG_ROOT}/include"
+      ~/Library/Frameworks
+      /Library/Frameworks
+      /usr/local/include
+      /usr/include
+      /sw/include # Fink
+      /opt/local/include # DarwinPorts
+      /opt/csw/include # Blastwave
+      /opt/include
+      /usr/freeware/include
+    PATH_SUFFIXES
+      ffmpeg
+    DOC "FFMPEG's ${component} include directory")
+  mark_as_advanced("FFMPEG_${component}_INCLUDE_DIR")
+
+  # On Windows, static FFMPEG is sometimes built as `lib<name>.a`.
+  if (WIN32)
+    list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES ".a" ".lib")
+    list(APPEND CMAKE_FIND_LIBRARY_PREFIXES "" "lib")
+  endif ()
+
+  find_library("FFMPEG_${component}_LIBRARY"
+    NAMES
+      "${component}"
+    PATHS
+      "${FFMPEG_ROOT}/lib"
+      ~/Library/Frameworks
+      /Library/Frameworks
+      /usr/local/lib
+      /usr/local/lib64
+      /usr/lib
+      /usr/lib64
+      /sw/lib
+      /opt/local/lib
+      /opt/csw/lib
+      /opt/lib
+      /usr/freeware/lib64
+      "${FFMPEG_ROOT}/bin"
+    DOC "FFMPEG's ${component} library")
+  mark_as_advanced("FFMPEG_${component}_LIBRARY")
+
+  if (FFMPEG_${component}_LIBRARY AND FFMPEG_${component}_INCLUDE_DIR)
+    set(_deps_found TRUE)
+    set(_deps_link)
+    foreach (_ffmpeg_dep IN LISTS ARGN)
+      if (TARGET "FFMPEG::${_ffmpeg_dep}")
+        list(APPEND _deps_link "FFMPEG::${_ffmpeg_dep}")
+      else ()
+        set(_deps_found FALSE)
+      endif ()
+    endforeach ()
+    if (_deps_found)
+      add_library("FFMPEG::${component}" UNKNOWN IMPORTED)
+      set_target_properties("FFMPEG::${component}" PROPERTIES
+        IMPORTED_LOCATION "${FFMPEG_${component}_LIBRARY}"
+        INTERFACE_INCLUDE_DIRECTORIES "${FFMPEG_${component}_INCLUDE_DIR}"
+        IMPORTED_LINK_INTERFACE_LIBRARIES "${_deps_link}")
+      set("FFMPEG_${component}_FOUND" 1
+        PARENT_SCOPE)
+
+      set(version_header_path "${FFMPEG_${component}_INCLUDE_DIR}/lib${component}/version.h")
+      if (EXISTS "${version_header_path}")
+        string(TOUPPER "${component}" component_upper)
+        file(STRINGS "${version_header_path}" version
+          REGEX "#define  *LIB${component_upper}_VERSION_(MAJOR|MINOR|MICRO) ")
+        string(REGEX REPLACE ".*_MAJOR *\([0-9]*\).*" "\\1" major "${version}")
+        string(REGEX REPLACE ".*_MINOR *\([0-9]*\).*" "\\1" minor "${version}")
+        string(REGEX REPLACE ".*_MICRO *\([0-9]*\).*" "\\1" micro "${version}")
+        if (NOT major STREQUAL "" AND
+            NOT minor STREQUAL "" AND
+            NOT micro STREQUAL "")
+          set("FFMPEG_${component}_VERSION" "${major}.${minor}.${micro}"
+            PARENT_SCOPE)
+        endif ()
+      endif ()
+    else ()
+      set("FFMPEG_${component}_FOUND" 0
+        PARENT_SCOPE)
+      set(what)
+      if (NOT FFMPEG_${component}_LIBRARY)
+        set(what "library")
+      endif ()
+      if (NOT FFMPEG_${component}_INCLUDE_DIR)
+        if (what)
+          string(APPEND what " or headers")
+        else ()
+          set(what "headers")
+        endif ()
+      endif ()
+      set("FFMPEG_${component}_NOT_FOUND_MESSAGE"
+        "Could not find the ${what} for ${component}."
+        PARENT_SCOPE)
+    endif ()
+  endif ()
+endfunction ()
+
+_ffmpeg_find(avutil     avutil.h)
+_ffmpeg_find(avresample avresample.h
+  avutil)
+_ffmpeg_find(swresample swresample.h
+  avutil)
+_ffmpeg_find(swscale    swscale.h
+  avutil)
+_ffmpeg_find(avcodec    avcodec.h
+  avutil)
+_ffmpeg_find(avformat   avformat.h
+  avcodec avutil)
+_ffmpeg_find(avfilter   avfilter.h
+  avutil)
+_ffmpeg_find(avdevice   avdevice.h
+  avformat avutil)
+
+if (TARGET FFMPEG::avutil)
+  set(_ffmpeg_version_header_path "${FFMPEG_avutil_INCLUDE_DIR}/libavutil/ffversion.h")
+  if (EXISTS "${_ffmpeg_version_header_path}")
+    file(STRINGS "${_ffmpeg_version_header_path}" _ffmpeg_version
+      REGEX "FFMPEG_VERSION")
+    string(REGEX REPLACE ".*\"n?\(.*\)\"" "\\1" FFMPEG_VERSION "${_ffmpeg_version}")
+    unset(_ffmpeg_version)
+  else ()
+    set(FFMPEG_VERSION FFMPEG_VERSION-NOTFOUND)
+  endif ()
+  unset(_ffmpeg_version_header_path)
+endif ()
+
+set(FFMPEG_INCLUDE_DIRS)
+set(FFMPEG_LIBRARIES)
+set(_ffmpeg_required_vars)
+foreach (_ffmpeg_component IN LISTS FFMPEG_FIND_COMPONENTS)
+  if (TARGET "FFMPEG::${_ffmpeg_component}")
+    set(FFMPEG_${_ffmpeg_component}_INCLUDE_DIRS
+      "${FFMPEG_${_ffmpeg_component}_INCLUDE_DIR}")
+    set(FFMPEG_${_ffmpeg_component}_LIBRARIES
+      "${FFMPEG_${_ffmpeg_component}_LIBRARY}")
+    list(APPEND FFMPEG_INCLUDE_DIRS
+      "${FFMPEG_${_ffmpeg_component}_INCLUDE_DIRS}")
+    list(APPEND FFMPEG_LIBRARIES
+      "${FFMPEG_${_ffmpeg_component}_LIBRARIES}")
+    if (FFMEG_FIND_REQUIRED_${_ffmpeg_component})
+      list(APPEND _ffmpeg_required_vars
+        "FFMPEG_${_ffmpeg_required_vars}_INCLUDE_DIRS"
+        "FFMPEG_${_ffmpeg_required_vars}_LIBRARIES")
+    endif ()
+  endif ()
+endforeach ()
+unset(_ffmpeg_component)
+
+if (FFMPEG_INCLUDE_DIRS)
+  list(REMOVE_DUPLICATES FFMPEG_INCLUDE_DIRS)
+endif ()
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(FFMPEG
+  REQUIRED_VARS FFMPEG_INCLUDE_DIRS FFMPEG_LIBRARIES ${_ffmpeg_required_vars}
+  VERSION_VAR FFMPEG_VERSION
+  HANDLE_COMPONENTS)
+unset(_ffmpeg_required_vars)