core/dumping: Add FFmpeg implementation

Sorry for the large diff, the implementation is quite long, and I can't really find a good way to split it into commits.

src/core/dumping/ffmpeg_backend.cpp

@@ -0,0 +1,530 @@
+// Copyright 2018 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#include "common/assert.h"
+#include "common/file_util.h"
+#include "common/logging/log.h"
+#include "core/dumping/ffmpeg_backend.h"
+#include "video_core/renderer_base.h"
+#include "video_core/video_core.h"
+
+extern "C" {
+#include <libavutil/opt.h>
+}
+
+namespace VideoDumper {
+
+void InitializeFFmpegLibraries() {
+    static bool initialized = false;
+
+    if (initialized)
+        return;
+#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 9, 100)
+    av_register_all();
+#endif
+    avformat_network_init();
+    initialized = true;
+}
+
+FFmpegStream::~FFmpegStream() {
+    Free();
+}
+
+bool FFmpegStream::Init(AVFormatContext* format_context_) {
+    InitializeFFmpegLibraries();
+
+    format_context = format_context_;
+    return true;
+}
+
+void FFmpegStream::Free() {
+    codec_context.reset();
+}
+
+void FFmpegStream::Flush() {
+    SendFrame(nullptr);
+}
+
+void FFmpegStream::WritePacket(AVPacket& packet) {
+    if (packet.pts != static_cast<s64>(AV_NOPTS_VALUE)) {
+        packet.pts = av_rescale_q(packet.pts, codec_context->time_base, stream->time_base);
+    }
+    if (packet.dts != static_cast<s64>(AV_NOPTS_VALUE)) {
+        packet.dts = av_rescale_q(packet.dts, codec_context->time_base, stream->time_base);
+    }
+    packet.stream_index = stream->index;
+    av_interleaved_write_frame(format_context, &packet);
+}
+
+void FFmpegStream::SendFrame(AVFrame* frame) {
+    // Initialize packet
+    AVPacket packet;
+    av_init_packet(&packet);
+    packet.data = nullptr;
+    packet.size = 0;
+
+    // Encode frame
+    if (avcodec_send_frame(codec_context.get(), frame) < 0) {
+        LOG_ERROR(Render, "Frame dropped: could not send frame");
+        return;
+    }
+    int error = 1;
+    while (error >= 0) {
+        error = avcodec_receive_packet(codec_context.get(), &packet);
+        if (error == AVERROR(EAGAIN) || error == AVERROR_EOF)
+            return;
+        if (error < 0) {
+            LOG_ERROR(Render, "Frame dropped: could not encode audio");
+            return;
+        } else {
+            // Write frame to video file
+            WritePacket(packet);
+        }
+    }
+}
+
+FFmpegVideoStream::~FFmpegVideoStream() {
+    Free();
+}
+
+bool FFmpegVideoStream::Init(AVFormatContext* format_context, AVOutputFormat* output_format,
+                             const Layout::FramebufferLayout& layout_) {
+
+    InitializeFFmpegLibraries();
+
+    if (!FFmpegStream::Init(format_context))
+        return false;
+
+    layout = layout_;
+    frame_count = 0;
+
+    // Initialize video codec
+    // Ensure VP9 codec here, also to avoid patent issues
+    constexpr AVCodecID codec_id = AV_CODEC_ID_VP9;
+    const AVCodec* codec = avcodec_find_encoder(codec_id);
+    codec_context.reset(avcodec_alloc_context3(codec));
+    if (!codec || !codec_context) {
+        LOG_ERROR(Render, "Could not find video encoder or allocate video codec context");
+        return false;
+    }
+
+    // Configure video codec context
+    codec_context->codec_type = AVMEDIA_TYPE_VIDEO;
+    codec_context->bit_rate = 2500000;
+    codec_context->width = layout.width;
+    codec_context->height = layout.height;
+    codec_context->time_base.num = 1;
+    codec_context->time_base.den = 60;
+    codec_context->gop_size = 12;
+    codec_context->pix_fmt = AV_PIX_FMT_YUV420P;
+    codec_context->thread_count = 8;
+    if (output_format->flags & AVFMT_GLOBALHEADER)
+        codec_context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
+    av_opt_set_int(codec_context.get(), "cpu-used", 5, 0);
+
+    if (avcodec_open2(codec_context.get(), codec, nullptr) < 0) {
+        LOG_ERROR(Render, "Could not open video codec");
+        return false;
+    }
+
+    // Create video stream
+    stream = avformat_new_stream(format_context, codec);
+    if (!stream || avcodec_parameters_from_context(stream->codecpar, codec_context.get()) < 0) {
+        LOG_ERROR(Render, "Could not create video stream");
+        return false;
+    }
+
+    // Allocate frames
+    current_frame.reset(av_frame_alloc());
+    scaled_frame.reset(av_frame_alloc());
+    scaled_frame->format = codec_context->pix_fmt;
+    scaled_frame->width = layout.width;
+    scaled_frame->height = layout.height;
+    if (av_frame_get_buffer(scaled_frame.get(), 1) < 0) {
+        LOG_ERROR(Render, "Could not allocate frame buffer");
+        return false;
+    }
+
+    // Create SWS Context
+    auto* context = sws_getCachedContext(
+        sws_context.get(), layout.width, layout.height, pixel_format, layout.width, layout.height,
+        codec_context->pix_fmt, SWS_BICUBIC, nullptr, nullptr, nullptr);
+    if (context != sws_context.get())
+        sws_context.reset(context);
+
+    return true;
+}
+
+void FFmpegVideoStream::Free() {
+    FFmpegStream::Free();
+
+    current_frame.reset();
+    scaled_frame.reset();
+    sws_context.reset();
+}
+
+void FFmpegVideoStream::ProcessFrame(VideoFrame& frame) {
+    if (frame.width != layout.width || frame.height != layout.height) {
+        LOG_ERROR(Render, "Frame dropped: resolution does not match");
+        return;
+    }
+    // Prepare frame
+    current_frame->data[0] = frame.data.data();
+    current_frame->linesize[0] = frame.stride;
+    current_frame->format = pixel_format;
+    current_frame->width = layout.width;
+    current_frame->height = layout.height;
+
+    // Scale the frame
+    if (sws_context) {
+        sws_scale(sws_context.get(), current_frame->data, current_frame->linesize, 0, layout.height,
+                  scaled_frame->data, scaled_frame->linesize);
+    }
+    scaled_frame->pts = frame_count++;
+
+    // Encode frame
+    SendFrame(scaled_frame.get());
+}
+
+FFmpegAudioStream::~FFmpegAudioStream() {
+    Free();
+}
+
+bool FFmpegAudioStream::Init(AVFormatContext* format_context) {
+    InitializeFFmpegLibraries();
+
+    if (!FFmpegStream::Init(format_context))
+        return false;
+
+    sample_count = 0;
+
+    // Initialize audio codec
+    constexpr AVCodecID codec_id = AV_CODEC_ID_VORBIS;
+    const AVCodec* codec = avcodec_find_encoder(codec_id);
+    codec_context.reset(avcodec_alloc_context3(codec));
+    if (!codec || !codec_context) {
+        LOG_ERROR(Render, "Could not find audio encoder or allocate audio codec context");
+        return false;
+    }
+
+    // Configure audio codec context
+    codec_context->codec_type = AVMEDIA_TYPE_AUDIO;
+    codec_context->bit_rate = 64000;
+    codec_context->sample_fmt = codec->sample_fmts[0];
+    codec_context->sample_rate = AudioCore::native_sample_rate;
+    codec_context->channel_layout = AV_CH_LAYOUT_STEREO;
+    codec_context->channels = 2;
+
+    if (avcodec_open2(codec_context.get(), codec, nullptr) < 0) {
+        LOG_ERROR(Render, "Could not open audio codec");
+        return false;
+    }
+
+    // Create audio stream
+    stream = avformat_new_stream(format_context, codec);
+    if (!stream || avcodec_parameters_from_context(stream->codecpar, codec_context.get()) < 0) {
+
+        LOG_ERROR(Render, "Could not create audio stream");
+        return false;
+    }
+
+    // Allocate frame
+    audio_frame.reset(av_frame_alloc());
+    audio_frame->format = codec_context->sample_fmt;
+    audio_frame->channel_layout = codec_context->channel_layout;
+    audio_frame->channels = codec_context->channels;
+
+    // Allocate SWR context
+    auto* context =
+        swr_alloc_set_opts(nullptr, codec_context->channel_layout, codec_context->sample_fmt,
+                           codec_context->sample_rate, codec_context->channel_layout,
+                           AV_SAMPLE_FMT_S16P, AudioCore::native_sample_rate, 0, nullptr);
+    if (!context) {
+        LOG_ERROR(Render, "Could not create SWR context");
+        return false;
+    }
+    swr_context.reset(context);
+    if (swr_init(swr_context.get()) < 0) {
+        LOG_ERROR(Render, "Could not init SWR context");
+        return false;
+    }
+
+    // Allocate resampled data
+    int error =
+        av_samples_alloc_array_and_samples(&resampled_data, nullptr, codec_context->channels,
+                                           codec_context->frame_size, codec_context->sample_fmt, 0);
+    if (error < 0) {
+        LOG_ERROR(Render, "Could not allocate samples storage");
+        return false;
+    }
+
+    return true;
+}
+
+void FFmpegAudioStream::Free() {
+    FFmpegStream::Free();
+
+    audio_frame.reset();
+    swr_context.reset();
+    // Free resampled data
+    if (resampled_data) {
+        av_freep(&resampled_data[0]);
+    }
+    av_freep(&resampled_data);
+}
+
+void FFmpegAudioStream::ProcessFrame(VariableAudioFrame& channel0, VariableAudioFrame& channel1) {
+    ASSERT_MSG(channel0.size() == channel1.size(),
+               "Frames of the two channels must have the same number of samples");
+    std::array<const u8*, 2> src_data = {reinterpret_cast<u8*>(channel0.data()),
+                                         reinterpret_cast<u8*>(channel1.data())};
+    if (swr_convert(swr_context.get(), resampled_data, channel0.size(), src_data.data(),
+                    channel0.size()) < 0) {
+
+        LOG_ERROR(Render, "Audio frame dropped: Could not resample data");
+        return;
+    }
+
+    // Prepare frame
+    audio_frame->nb_samples = channel0.size();
+    audio_frame->data[0] = resampled_data[0];
+    audio_frame->data[1] = resampled_data[1];
+    audio_frame->pts = sample_count;
+    sample_count += channel0.size();
+
+    SendFrame(audio_frame.get());
+}
+
+std::size_t FFmpegAudioStream::GetAudioFrameSize() const {
+    ASSERT_MSG(codec_context, "Codec context is not initialized yet!");
+    return codec_context->frame_size;
+}
+
+FFmpegMuxer::~FFmpegMuxer() {
+    Free();
+}
+
+bool FFmpegMuxer::Init(const std::string& path, const std::string& format,
+                       const Layout::FramebufferLayout& layout) {
+
+    InitializeFFmpegLibraries();
+
+    if (!FileUtil::CreateFullPath(path)) {
+        return false;
+    }
+
+    // Get output format
+    // Ensure webm here to avoid patent issues
+    ASSERT_MSG(format == "webm", "Only webm is allowed for frame dumping");
+    auto* output_format = av_guess_format(format.c_str(), path.c_str(), "video/webm");
+    if (!output_format) {
+        LOG_ERROR(Render, "Could not get format {}", format);
+        return false;
+    }
+
+    // Initialize format context
+    auto* format_context_raw = format_context.get();
+    if (avformat_alloc_output_context2(&format_context_raw, output_format, nullptr, path.c_str()) <
+        0) {
+
+        LOG_ERROR(Render, "Could not allocate output context");
+        return false;
+    }
+    format_context.reset(format_context_raw);
+
+    if (!video_stream.Init(format_context.get(), output_format, layout))
+        return false;
+    if (!audio_stream.Init(format_context.get()))
+        return false;
+
+    // Open video file
+    if (avio_open(&format_context->pb, path.c_str(), AVIO_FLAG_WRITE) < 0 ||
+        avformat_write_header(format_context.get(), nullptr)) {
+
+        LOG_ERROR(Render, "Could not open {}", path);
+        return false;
+    }
+
+    LOG_INFO(Render, "Dumping frames to {} ({}x{})", path, layout.width, layout.height);
+    return true;
+}
+
+void FFmpegMuxer::Free() {
+    video_stream.Free();
+    audio_stream.Free();
+    format_context.reset();
+}
+
+void FFmpegMuxer::ProcessVideoFrame(VideoFrame& frame) {
+    video_stream.ProcessFrame(frame);
+}
+
+void FFmpegMuxer::ProcessAudioFrame(VariableAudioFrame& channel0, VariableAudioFrame& channel1) {
+    audio_stream.ProcessFrame(channel0, channel1);
+}
+
+void FFmpegMuxer::FlushVideo() {
+    video_stream.Flush();
+}
+
+void FFmpegMuxer::FlushAudio() {
+    audio_stream.Flush();
+}
+
+std::size_t FFmpegMuxer::GetAudioFrameSize() const {
+    return audio_stream.GetAudioFrameSize();
+}
+
+void FFmpegMuxer::WriteTrailer() {
+    av_write_trailer(format_context.get());
+}
+
+FFmpegBackend::FFmpegBackend() = default;
+
+FFmpegBackend::~FFmpegBackend() {
+    ASSERT_MSG(!IsDumping(), "Dumping must be stopped first");
+
+    if (video_processing_thread.joinable())
+        video_processing_thread.join();
+    if (audio_processing_thread.joinable())
+        audio_processing_thread.join();
+    ffmpeg.Free();
+}
+
+bool FFmpegBackend::StartDumping(const std::string& path, const std::string& format,
+                                 const Layout::FramebufferLayout& layout) {
+
+    InitializeFFmpegLibraries();
+
+    if (!ffmpeg.Init(path, format, layout)) {
+        ffmpeg.Free();
+        return false;
+    }
+
+    video_layout = layout;
+
+    if (video_processing_thread.joinable())
+        video_processing_thread.join();
+    video_processing_thread = std::thread([&] {
+        event1.Set();
+        while (true) {
+            event2.Wait();
+            current_buffer = (current_buffer + 1) % 2;
+            next_buffer = (current_buffer + 1) % 2;
+            event1.Set();
+            // Process this frame
+            auto& frame = video_frame_buffers[current_buffer];
+            if (frame.width == 0 && frame.height == 0) {
+                // An empty frame marks the end of frame data
+                ffmpeg.FlushVideo();
+                break;
+            }
+            ffmpeg.ProcessVideoFrame(frame);
+        }
+        // Finish audio execution first if not done yet
+        if (audio_processing_thread.joinable())
+            audio_processing_thread.join();
+        EndDumping();
+    });
+
+    if (audio_processing_thread.joinable())
+        audio_processing_thread.join();
+    audio_processing_thread = std::thread([&] {
+        VariableAudioFrame channel0, channel1;
+        while (true) {
+            channel0 = audio_frame_queues[0].PopWait();
+            channel1 = audio_frame_queues[1].PopWait();
+            if (channel0.empty()) {
+                // An empty frame marks the end of frame data
+                ffmpeg.FlushAudio();
+                break;
+            }
+            ffmpeg.ProcessAudioFrame(channel0, channel1);
+        }
+    });
+
+    VideoCore::g_renderer->PrepareVideoDumping();
+    is_dumping = true;
+
+    return true;
+}
+
+void FFmpegBackend::AddVideoFrame(const VideoFrame& frame) {
+    event1.Wait();
+    video_frame_buffers[next_buffer] = std::move(frame);
+    event2.Set();
+}
+
+void FFmpegBackend::AddAudioFrame(const AudioCore::StereoFrame16& frame) {
+    std::array<std::array<s16, 160>, 2> refactored_frame;
+    for (std::size_t i = 0; i < frame.size(); i++) {
+        refactored_frame[0][i] = frame[i][0];
+        refactored_frame[1][i] = frame[i][1];
+    }
+
+    for (auto i : {0, 1}) {
+        audio_buffers[i].insert(audio_buffers[i].end(), refactored_frame[i].begin(),
+                                refactored_frame[i].end());
+    }
+    CheckAudioBuffer();
+}
+
+void FFmpegBackend::AddAudioSample(const std::array<s16, 2>& sample) {
+    for (auto i : {0, 1}) {
+        audio_buffers[i].push_back(sample[i]);
+    }
+    CheckAudioBuffer();
+}
+
+void FFmpegBackend::StopDumping() {
+    is_dumping = false;
+    VideoCore::g_renderer->CleanupVideoDumping();
+
+    // Flush the video processing queue
+    AddVideoFrame(VideoFrame());
+    for (auto i : {0, 1}) {
+        // Add remaining data to audio queue
+        if (audio_buffers[i].size() >= 0) {
+            VariableAudioFrame buffer(audio_buffers[i].begin(), audio_buffers[i].end());
+            audio_frame_queues[i].Push(std::move(buffer));
+            audio_buffers[i].clear();
+        }
+        // Flush the audio processing queue
+        audio_frame_queues[i].Push(VariableAudioFrame());
+    }
+    // Wait until processing ends
+    processing_ended.Wait();
+}
+
+bool FFmpegBackend::IsDumping() const {
+    return is_dumping.load(std::memory_order_relaxed);
+}
+
+Layout::FramebufferLayout FFmpegBackend::GetLayout() const {
+    return video_layout;
+}
+
+void FFmpegBackend::EndDumping() {
+    LOG_INFO(Render, "Ending frame dumping");
+
+    ffmpeg.WriteTrailer();
+    ffmpeg.Free();
+    processing_ended.Set();
+}
+
+void FFmpegBackend::CheckAudioBuffer() {
+    for (auto i : {0, 1}) {
+        const std::size_t frame_size = ffmpeg.GetAudioFrameSize();
+        // Add audio data to the queue when there is enough to form a frame
+        while (audio_buffers[i].size() >= frame_size) {
+            VariableAudioFrame buffer(audio_buffers[i].begin(),
+                                      audio_buffers[i].begin() + frame_size);
+            audio_frame_queues[i].Push(std::move(buffer));
+
+            audio_buffers[i].erase(audio_buffers[i].begin(), audio_buffers[i].begin() + frame_size);
+        }
+    }
+}
+
+} // namespace VideoDumper

src/core/dumping/ffmpeg_backend.h

@@ -0,0 +1,196 @@
+// Copyright 2018 Citra Emulator Project
+// Licensed under GPLv2 or any later version
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <atomic>
+#include <condition_variable>
+#include <limits>
+#include <memory>
+#include <mutex>
+#include <thread>
+#include <vector>
+#include "common/common_types.h"
+#include "common/thread.h"
+#include "common/threadsafe_queue.h"
+#include "core/dumping/backend.h"
+
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavformat/avformat.h>
+#include <libswresample/swresample.h>
+#include <libswscale/swscale.h>
+}
+
+namespace VideoDumper {
+
+using VariableAudioFrame = std::vector<s16>;
+
+void InitFFmpegLibraries();
+
+/**
+ * Wrapper around FFmpeg AVCodecContext + AVStream.
+ * Rescales/Resamples, encodes and writes a frame.
+ */
+class FFmpegStream {
+public:
+    bool Init(AVFormatContext* format_context);
+    void Free();
+    void Flush();
+
+protected:
+    ~FFmpegStream();
+
+    void WritePacket(AVPacket& packet);
+    void SendFrame(AVFrame* frame);
+
+    struct AVCodecContextDeleter {
+        void operator()(AVCodecContext* codec_context) const {
+            avcodec_free_context(&codec_context);
+        }
+    };
+
+    struct AVFrameDeleter {
+        void operator()(AVFrame* frame) const {
+            av_frame_free(&frame);
+        }
+    };
+
+    AVFormatContext* format_context{};
+    std::unique_ptr<AVCodecContext, AVCodecContextDeleter> codec_context{};
+    AVStream* stream{};
+};
+
+/**
+ * A FFmpegStream used for video data.
+ * Rescales, encodes and writes a frame.
+ */
+class FFmpegVideoStream : public FFmpegStream {
+public:
+    ~FFmpegVideoStream();
+
+    bool Init(AVFormatContext* format_context, AVOutputFormat* output_format,
+              const Layout::FramebufferLayout& layout);
+    void Free();
+    void ProcessFrame(VideoFrame& frame);
+
+private:
+    struct SwsContextDeleter {
+        void operator()(SwsContext* sws_context) const {
+            sws_freeContext(sws_context);
+        }
+    };
+
+    u64 frame_count{};
+
+    std::unique_ptr<AVFrame, AVFrameDeleter> current_frame{};
+    std::unique_ptr<AVFrame, AVFrameDeleter> scaled_frame{};
+    std::unique_ptr<SwsContext, SwsContextDeleter> sws_context{};
+    Layout::FramebufferLayout layout;
+
+    /// The pixel format the frames are stored in
+    static constexpr AVPixelFormat pixel_format = AVPixelFormat::AV_PIX_FMT_BGRA;
+};
+
+/**
+ * A FFmpegStream used for audio data.
+ * Resamples (converts), encodes and writes a frame.
+ */
+class FFmpegAudioStream : public FFmpegStream {
+public:
+    ~FFmpegAudioStream();
+
+    bool Init(AVFormatContext* format_context);
+    void Free();
+    void ProcessFrame(VariableAudioFrame& channel0, VariableAudioFrame& channel1);
+    std::size_t GetAudioFrameSize() const;
+
+private:
+    struct SwrContextDeleter {
+        void operator()(SwrContext* swr_context) const {
+            swr_free(&swr_context);
+        }
+    };
+
+    u64 sample_count{};
+
+    std::unique_ptr<AVFrame, AVFrameDeleter> audio_frame{};
+    std::unique_ptr<SwrContext, SwrContextDeleter> swr_context{};
+
+    u8** resampled_data{};
+};
+
+/**
+ * Wrapper around FFmpeg AVFormatContext.
+ * Manages the video and audio streams, and accepts video and audio data.
+ */
+class FFmpegMuxer {
+public:
+    ~FFmpegMuxer();
+
+    bool Init(const std::string& path, const std::string& format,
+              const Layout::FramebufferLayout& layout);
+    void Free();
+    void ProcessVideoFrame(VideoFrame& frame);
+    void ProcessAudioFrame(VariableAudioFrame& channel0, VariableAudioFrame& channel1);
+    void FlushVideo();
+    void FlushAudio();
+    std::size_t GetAudioFrameSize() const;
+    void WriteTrailer();
+
+private:
+    struct AVFormatContextDeleter {
+        void operator()(AVFormatContext* format_context) const {
+            avio_closep(&format_context->pb);
+            avformat_free_context(format_context);
+        }
+    };
+
+    FFmpegAudioStream audio_stream{};
+    FFmpegVideoStream video_stream{};
+    std::unique_ptr<AVFormatContext, AVFormatContextDeleter> format_context{};
+};
+
+/**
+ * FFmpeg video dumping backend.
+ * This class implements a double buffer, and an audio queue to keep audio data
+ * before enough data is received to form a frame.
+ */
+class FFmpegBackend : public Backend {
+public:
+    FFmpegBackend();
+    ~FFmpegBackend() override;
+    bool StartDumping(const std::string& path, const std::string& format,
+                      const Layout::FramebufferLayout& layout) override;
+    void AddVideoFrame(const VideoFrame& frame) override;
+    void AddAudioFrame(const AudioCore::StereoFrame16& frame) override;
+    void AddAudioSample(const std::array<s16, 2>& sample) override;
+    void StopDumping() override;
+    bool IsDumping() const override;
+    Layout::FramebufferLayout GetLayout() const override;
+
+private:
+    void CheckAudioBuffer();
+    void EndDumping();
+
+    std::atomic_bool is_dumping = false; ///< Whether the backend is currently dumping
+
+    FFmpegMuxer ffmpeg{};
+
+    Layout::FramebufferLayout video_layout;
+    std::array<VideoFrame, 2> video_frame_buffers;
+    u32 current_buffer = 0, next_buffer = 1;
+    Common::Event event1, event2;
+    std::thread video_processing_thread;
+
+    /// An audio buffer used to temporarily hold audio data, before the size is big enough
+    /// to be sent to the encoder as a frame
+    std::array<VariableAudioFrame, 2> audio_buffers;
+    std::array<Common::SPSCQueue<VariableAudioFrame>, 2> audio_frame_queues;
+    std::thread audio_processing_thread;
+
+    Common::Event processing_ended;
+};
+
+} // namespace VideoDumper