Revive: dumping/ffmpeg_backend: Various fixes (#6528)

* dumping/ffmpeg_backend: Add FPS filter

So that the recorded video can be at 60FPS (which is supported by most encoders) while still maintaining correct speed.

* dumping/ffmpeg_backend: Add HW context support

Required for some HW acceled encoders. Not tested as my devices don't seem to require this.

* CMake: Copy avfilter dll for MSVC

* CMakeLists: Require FFmpeg 4.0

* ffmpeg: Fix dumper compile error on MSVC.

* ffmpeg: Address review comments.

---------

Co-authored-by: zhupengfei <zhupf321@gmail.com>

CMakeLists.txt

@@ -255,12 +255,12 @@ if (ENABLE_FFMPEG)
     endif()
 
     if (ENABLE_FFMPEG_VIDEO_DUMPER)
-        find_package(FFmpeg REQUIRED COMPONENTS avcodec avformat avutil swscale swresample)
+        find_package(FFmpeg REQUIRED COMPONENTS avcodec avfilter avformat avutil swresample)
     else()
         find_package(FFmpeg REQUIRED COMPONENTS avcodec)
     endif()
-    if ("${FFmpeg_avcodec_VERSION}" VERSION_LESS "57.48.101")
+    if ("${FFmpeg_avcodec_VERSION}" VERSION_LESS "58.4.100")
-        message(FATAL_ERROR "Found version for libavcodec is too low. The required version is at least 57.48.101 (included in FFmpeg 3.1 and later).")
+        message(FATAL_ERROR "Found version for libavcodec is too low. The required version is at least 58.4.100 (included in FFmpeg 4.0 and later).")
     endif()
 endif()
 

CMakeModules/CopyCitraFFmpegDeps.cmake

@@ -3,8 +3,10 @@ function(copy_citra_FFmpeg_deps target_dir)
     set(DLL_DEST "${CMAKE_BINARY_DIR}/bin/$<CONFIG>/")
     windows_copy_files(${target_dir} ${FFMPEG_DIR}/bin ${DLL_DEST}
         avcodec*.dll
+        avfilter*.dll
         avformat*.dll
         avutil*.dll
+        postproc*.dll
         swresample*.dll
         swscale*.dll
     )

src/core/CMakeLists.txt

@@ -505,7 +505,7 @@ if ("x86_64" IN_LIST ARCHITECTURE OR "arm64" IN_LIST ARCHITECTURE)
 endif()
 
 if (ENABLE_FFMPEG_VIDEO_DUMPER)
-    target_link_libraries(citra_core PUBLIC FFmpeg::avcodec FFmpeg::avformat FFmpeg::swscale FFmpeg::swresample FFmpeg::avutil)
+    target_link_libraries(citra_core PUBLIC FFmpeg::avcodec FFmpeg::avfilter FFmpeg::avformat FFmpeg::swresample FFmpeg::avutil)
 endif()
 
 if (CITRA_USE_PRECOMPILED_HEADERS)

src/core/dumping/ffmpeg_backend.cpp

@@ -7,6 +7,7 @@
 #include "common/file_util.h"
 #include "common/logging/log.h"
 #include "common/param_package.h"
+#include "common/scope_exit.h"
 #include "common/settings.h"
 #include "common/string_util.h"
 #include "core/dumping/ffmpeg_backend.h"
@@ -15,6 +16,9 @@
 #include "video_core/video_core.h"
 
 extern "C" {
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
+#include <libavutil/hwcontext.h>
 #include <libavutil/pixdesc.h>
 }
 
@@ -102,6 +106,43 @@ FFmpegVideoStream::~FFmpegVideoStream() {
     Free();
 }
 
+// This is modified from libavcodec/decode.c
+// The original version was broken
+static AVPixelFormat GetPixelFormat(AVCodecContext* avctx, const AVPixelFormat* fmt) {
+    // Choose a software pixel format if any, prefering those in the front of the list
+    for (int i = 0; fmt[i] != AV_PIX_FMT_NONE; i++) {
+        const AVPixFmtDescriptor* desc = av_pix_fmt_desc_get(fmt[i]);
+        if (!(desc->flags & AV_PIX_FMT_FLAG_HWACCEL)) {
+            return fmt[i];
+        }
+    }
+
+    // Finally, traverse the list in order and choose the first entry
+    // with no external dependencies (if there is no hardware configuration
+    // information available then this just picks the first entry).
+    for (int i = 0; fmt[i] != AV_PIX_FMT_NONE; i++) {
+        const AVCodecHWConfig* config;
+        for (int j = 0;; j++) {
+            config = avcodec_get_hw_config(avctx->codec, j);
+            if (!config || config->pix_fmt == fmt[i]) {
+                break;
+            }
+        }
+        if (!config) {
+            // No specific config available, so the decoder must be able
+            // to handle this format without any additional setup.
+            return fmt[i];
+        }
+        if (config->methods & AV_CODEC_HW_CONFIG_METHOD_INTERNAL) {
+            // Usable with only internal setup.
+            return fmt[i];
+        }
+    }
+
+    // Nothing is usable, give up.
+    return AV_PIX_FMT_NONE;
+}
+
 bool FFmpegVideoStream::Init(FFmpegMuxer& muxer, const Layout::FramebufferLayout& layout_) {
 
     InitializeFFmpegLibraries();
@@ -125,15 +166,28 @@ bool FFmpegVideoStream::Init(FFmpegMuxer& muxer, const Layout::FramebufferLayout
     codec_context->bit_rate = Settings::values.video_bitrate;
     codec_context->width = layout.width;
     codec_context->height = layout.height;
-    // TODO(xperia64): While these numbers from core timing work fine, certain video codecs do not
+    // Use 60fps here, since the video is already filtered (resampled)
-    // support the strange resulting timebase (280071/16756991); Addressing this issue would require
+    codec_context->time_base.num = 1;
-    // resampling the video
+    codec_context->time_base.den = 60;
-    // List of codecs known broken by this change: mpeg1, mpeg2, mpeg4, libxvid
-    // See https://github.com/citra-emu/citra/pull/5273#issuecomment-643023325 for more information
-    codec_context->time_base.num = static_cast<int>(GPU::frame_ticks);
-    codec_context->time_base.den = static_cast<int>(BASE_CLOCK_RATE_ARM11);
     codec_context->gop_size = 12;
-    codec_context->pix_fmt = codec->pix_fmts ? codec->pix_fmts[0] : AV_PIX_FMT_YUV420P;
+
+    // Get pixel format for codec
+    if (codec->pix_fmts) {
+        sw_pixel_format = GetPixelFormat(codec_context.get(), codec->pix_fmts);
+    } else {
+        sw_pixel_format = AV_PIX_FMT_YUV420P;
+    }
+    if (sw_pixel_format == AV_PIX_FMT_NONE) {
+        // This encoder requires HW context configuration.
+        if (!InitHWContext(codec)) {
+            LOG_ERROR(Render, "Failed to initialize HW context");
+            return false;
+        }
+    } else {
+        requires_hw_frames = false;
+        codec_context->pix_fmt = sw_pixel_format;
+    }
+
     if (format_context->oformat->flags & AVFMT_GLOBALHEADER)
         codec_context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;
 
@@ -160,31 +214,28 @@ bool FFmpegVideoStream::Init(FFmpegMuxer& muxer, const Layout::FramebufferLayout
 
     // Allocate frames
     current_frame.reset(av_frame_alloc());
-    scaled_frame.reset(av_frame_alloc());
+    filtered_frame.reset(av_frame_alloc());
-    scaled_frame->format = codec_context->pix_fmt;
+
-    scaled_frame->width = layout.width;
+    if (requires_hw_frames) {
-    scaled_frame->height = layout.height;
+        hw_frame.reset(av_frame_alloc());
-    if (av_frame_get_buffer(scaled_frame.get(), 0) < 0) {
+        if (av_hwframe_get_buffer(codec_context->hw_frames_ctx, hw_frame.get(), 0) < 0) {
-        LOG_ERROR(Render, "Could not allocate frame buffer");
+            LOG_ERROR(Render, "Could not allocate buffer for HW frame");
             return false;
         }
+    }
 
-    // Create SWS Context
+    return InitFilters();
-    auto* context = sws_getCachedContext(
-        sws_context.get(), layout.width, layout.height, pixel_format, layout.width, layout.height,
-        codec_context->pix_fmt, SWS_BICUBIC, nullptr, nullptr, nullptr);
-    if (context != sws_context.get())
-        sws_context.reset(context);
-
-    return true;
 }
 
 void FFmpegVideoStream::Free() {
     FFmpegStream::Free();
 
     current_frame.reset();
-    scaled_frame.reset();
+    filtered_frame.reset();
-    sws_context.reset();
+    hw_frame.reset();
+    filter_graph.reset();
+    source_context = nullptr;
+    sink_context = nullptr;
 }
 
 void FFmpegVideoStream::ProcessFrame(VideoFrame& frame) {
@@ -198,20 +249,189 @@ void FFmpegVideoStream::ProcessFrame(VideoFrame& frame) {
     current_frame->format = pixel_format;
     current_frame->width = layout.width;
     current_frame->height = layout.height;
+    current_frame->pts = frame_count++;
 
-    // Scale the frame
+    // Filter the frame
-    if (av_frame_make_writable(scaled_frame.get()) < 0) {
+    if (av_buffersrc_add_frame(source_context, current_frame.get()) < 0) {
-        LOG_ERROR(Render, "Video frame dropped: Could not prepare frame");
+        LOG_ERROR(Render, "Video frame dropped: Could not add frame to filter graph");
         return;
     }
-    if (sws_context) {
+    while (true) {
-        sws_scale(sws_context.get(), current_frame->data, current_frame->linesize, 0, layout.height,
+        const int error = av_buffersink_get_frame(sink_context, filtered_frame.get());
-                  scaled_frame->data, scaled_frame->linesize);
+        if (error == AVERROR(EAGAIN) || error == AVERROR_EOF) {
+            return;
+        }
+        if (error < 0) {
+            LOG_ERROR(Render, "Video frame dropped: Could not receive frame from filter graph");
+            return;
+        } else {
+            if (requires_hw_frames) {
+                if (av_hwframe_transfer_data(hw_frame.get(), filtered_frame.get(), 0) < 0) {
+                    LOG_ERROR(Render, "Video frame dropped: Could not upload to HW frame");
+                    return;
+                }
+                SendFrame(hw_frame.get());
+            } else {
+                SendFrame(filtered_frame.get());
             }
-    scaled_frame->pts = frame_count++;
 
-    // Encode frame
+            av_frame_unref(filtered_frame.get());
-    SendFrame(scaled_frame.get());
+        }
+    }
+}
+
+bool FFmpegVideoStream::InitHWContext(const AVCodec* codec) {
+    for (std::size_t i = 0; codec->pix_fmts[i] != AV_PIX_FMT_NONE; ++i) {
+        const AVCodecHWConfig* config;
+        for (int j = 0;; ++j) {
+            config = avcodec_get_hw_config(codec, j);
+            if (!config || config->pix_fmt == codec->pix_fmts[i]) {
+                break;
+            }
+        }
+        // If we are at this point, there should not be any possible HW format that does not
+        // need configuration.
+        ASSERT_MSG(config, "HW pixel format that does not need config should have been selected");
+
+        if (!(config->methods & (AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX |
+                                 AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX))) {
+            // Maybe this format requires ad-hoc configuration, unsupported.
+            continue;
+        }
+
+        codec_context->pix_fmt = codec->pix_fmts[i];
+
+        // Create HW device context
+        AVBufferRef* hw_device_context;
+        SCOPE_EXIT({ av_buffer_unref(&hw_device_context); });
+
+        // TODO: Provide the argument here somehow.
+        // This is necessary for some devices like CUDA where you must supply the GPU name.
+        // This is not necessary for VAAPI, etc.
+        if (av_hwdevice_ctx_create(&hw_device_context, config->device_type, nullptr, nullptr, 0) <
+            0) {
+            LOG_ERROR(Render, "Failed to create HW device context");
+            continue;
+        }
+        codec_context->hw_device_ctx = av_buffer_ref(hw_device_context);
+
+        // Get the SW format
+        AVHWFramesConstraints* constraints =
+            av_hwdevice_get_hwframe_constraints(hw_device_context, nullptr);
+        SCOPE_EXIT({ av_hwframe_constraints_free(&constraints); });
+
+        if (constraints) {
+            sw_pixel_format = constraints->valid_sw_formats ? constraints->valid_sw_formats[0]
+                                                            : AV_PIX_FMT_YUV420P;
+        } else {
+            LOG_WARNING(Render, "Could not query HW device constraints");
+            sw_pixel_format = AV_PIX_FMT_YUV420P;
+        }
+
+        // For encoders that only need the HW device
+        if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) {
+            requires_hw_frames = false;
+            return true;
+        }
+
+        requires_hw_frames = true;
+
+        // Create HW frames context
+        AVBufferRef* hw_frames_context_ref;
+        SCOPE_EXIT({ av_buffer_unref(&hw_frames_context_ref); });
+
+        if (!(hw_frames_context_ref = av_hwframe_ctx_alloc(hw_device_context))) {
+            LOG_ERROR(Render, "Failed to create HW frames context");
+            continue;
+        }
+
+        AVHWFramesContext* hw_frames_context =
+            reinterpret_cast<AVHWFramesContext*>(hw_frames_context_ref->data);
+        hw_frames_context->format = codec->pix_fmts[i];
+        hw_frames_context->sw_format = sw_pixel_format;
+        hw_frames_context->width = codec_context->width;
+        hw_frames_context->height = codec_context->height;
+        hw_frames_context->initial_pool_size = 20; // value from FFmpeg's example
+
+        if (av_hwframe_ctx_init(hw_frames_context_ref) < 0) {
+            LOG_ERROR(Render, "Failed to initialize HW frames context");
+            continue;
+        }
+
+        codec_context->hw_frames_ctx = av_buffer_ref(hw_frames_context_ref);
+        return true;
+    }
+
+    LOG_ERROR(Render, "Failed to find a usable HW pixel format");
+    return false;
+}
+
+bool FFmpegVideoStream::InitFilters() {
+    filter_graph.reset(avfilter_graph_alloc());
+
+    const AVFilter* source = avfilter_get_by_name("buffer");
+    const AVFilter* sink = avfilter_get_by_name("buffersink");
+    if (!source || !sink) {
+        LOG_ERROR(Render, "Could not find buffer source or sink");
+        return false;
+    }
+
+    // Configure buffer source
+    static constexpr AVRational src_time_base{static_cast<int>(GPU::frame_ticks),
+                                              static_cast<int>(BASE_CLOCK_RATE_ARM11)};
+    const std::string in_args = fmt::format(
+        "video_size={}x{}:pix_fmt={}:time_base={}/{}:pixel_aspect=1", codec_context->width,
+        codec_context->height, pixel_format, src_time_base.num, src_time_base.den);
+    if (avfilter_graph_create_filter(&source_context, source, "in", in_args.c_str(), nullptr,
+                                     filter_graph.get()) < 0) {
+        LOG_ERROR(Render, "Could not create buffer source");
+        return false;
+    }
+
+    // Configure buffer sink
+    if (avfilter_graph_create_filter(&sink_context, sink, "out", nullptr, nullptr,
+                                     filter_graph.get()) < 0) {
+        LOG_ERROR(Render, "Could not create buffer sink");
+        return false;
+    }
+    const AVPixelFormat pix_fmts[] = {sw_pixel_format, AV_PIX_FMT_NONE};
+    if (av_opt_set_int_list(sink_context, "pix_fmts", pix_fmts, AV_PIX_FMT_NONE,
+                            AV_OPT_SEARCH_CHILDREN) < 0) {
+        LOG_ERROR(Render, "Could not set output pixel format");
+        return false;
+    }
+
+    // Initialize filter graph
+    // `outputs` as in outputs of the 'previous' graphs
+    AVFilterInOut* outputs = avfilter_inout_alloc();
+    outputs->name = av_strdup("in");
+    outputs->filter_ctx = source_context;
+    outputs->pad_idx = 0;
+    outputs->next = nullptr;
+
+    // `inputs` as in inputs to the 'next' graphs
+    AVFilterInOut* inputs = avfilter_inout_alloc();
+    inputs->name = av_strdup("out");
+    inputs->filter_ctx = sink_context;
+    inputs->pad_idx = 0;
+    inputs->next = nullptr;
+
+    SCOPE_EXIT({
+        avfilter_inout_free(&outputs);
+        avfilter_inout_free(&inputs);
+    });
+
+    if (avfilter_graph_parse_ptr(filter_graph.get(), filter_graph_desc.data(), &inputs, &outputs,
+                                 nullptr) < 0) {
+        LOG_ERROR(Render, "Could not parse or create filter graph");
+        return false;
+    }
+    if (avfilter_graph_config(filter_graph.get(), nullptr) < 0) {
+        LOG_ERROR(Render, "Could not configure filter graph");
+        return false;
+    }
+
+    return true;
 }
 
 FFmpegAudioStream::~FFmpegAudioStream() {

src/core/dumping/ffmpeg_backend.h

@@ -19,10 +19,10 @@
 
 extern "C" {
 #include <libavcodec/avcodec.h>
+#include <libavfilter/avfilter.h>
 #include <libavformat/avformat.h>
 #include <libavutil/opt.h>
 #include <libswresample/swresample.h>
-#include <libswscale/swscale.h>
 }
 
 namespace VideoDumper {
@@ -69,7 +69,7 @@ protected:
 
 /**
  * A FFmpegStream used for video data.
- * Rescales, encodes and writes a frame.
+ * Filters (scales), encodes and writes a frame.
  */
 class FFmpegVideoStream : public FFmpegStream {
 public:
@@ -80,21 +80,39 @@ public:
     void ProcessFrame(VideoFrame& frame);
 
 private:
-    struct SwsContextDeleter {
+    bool InitHWContext(const AVCodec* codec);
-        void operator()(SwsContext* sws_context) const {
+    bool InitFilters();
-            sws_freeContext(sws_context);
-        }
-    };
 
     u64 frame_count{};
 
     std::unique_ptr<AVFrame, AVFrameDeleter> current_frame{};
-    std::unique_ptr<AVFrame, AVFrameDeleter> scaled_frame{};
+    std::unique_ptr<AVFrame, AVFrameDeleter> filtered_frame{};
-    std::unique_ptr<SwsContext, SwsContextDeleter> sws_context{};
+    std::unique_ptr<AVFrame, AVFrameDeleter> hw_frame{};
     Layout::FramebufferLayout layout;
 
-    /// The pixel format the frames are stored in
+    /// The pixel format the input frames are stored in
     static constexpr AVPixelFormat pixel_format = AVPixelFormat::AV_PIX_FMT_BGRA;
+
+    // Software pixel format. For normal encoders, this is the format they accept. For HW-acceled
+    // encoders, this is the format the HW frames context accepts.
+    AVPixelFormat sw_pixel_format = AV_PIX_FMT_NONE;
+
+    /// Whether the encoder we are using requires HW frames to be supplied.
+    bool requires_hw_frames = false;
+
+    // Filter related
+    struct AVFilterGraphDeleter {
+        void operator()(AVFilterGraph* filter_graph) const {
+            avfilter_graph_free(&filter_graph);
+        }
+    };
+    std::unique_ptr<AVFilterGraph, AVFilterGraphDeleter> filter_graph{};
+    // These don't need to be freed apparently
+    AVFilterContext* source_context;
+    AVFilterContext* sink_context;
+
+    /// The filter graph to use. This graph means 'change FPS to 60, convert format if needed'
+    static constexpr std::string_view filter_graph_desc = "fps=60";
 };
 
 /**