Pgorszkowski/2.46/request video frame callback for holepunch westeros and rialto

Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp

@@ -4894,6 +4894,9 @@ WTFLogChannel& MediaPlayerPrivateGStreamer::logChannel() const
 std::optional<VideoFrameMetadata> MediaPlayerPrivateGStreamer::videoFrameMetadata()
 {
     Locker sampleLocker { m_sampleMutex };
+    if (isHolePunchRenderingEnabled() && m_videoSink)
+        return GStreamerQuirksManager::singleton().videoFrameMetadata(m_videoSink, m_lastVideoFrameMetadataSampleCount);
+
     if (!GST_IS_SAMPLE(m_sample.get()))
         return { };
 

Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.cpp

@@ -58,6 +58,55 @@ bool GStreamerHolePunchQuirkRialto::setHolePunchVideoRectangle(GstElement* video
     return true;
 }
 
+std::optional<VideoFrameMetadata> GStreamerHolePunchQuirkRialto::videoFrameMetadata(GRefPtr<GstElement> videoSink, uint64_t& lastVideoFrameMetadataSampleCount)
+{
+    if (UNLIKELY(!gstObjectHasProperty(videoSink.get(), "video_pts")
+        || !gstObjectHasProperty(videoSink.get(), "stats")))
+        return { };
+
+    gint64 pts90kHz = 0;
+    GstStructure* stats = nullptr;
+    g_object_get(videoSink.get(), "video_pts", &pts90kHz, "stats", &stats, nullptr);
+
+    if (!pts90kHz || !stats)
+        return { };
+
+    guint64 rendered = 0;
+    gst_structure_get_uint64(stats, "rendered", &rendered);
+    gst_structure_free(stats);
+
+    if (rendered == lastVideoFrameMetadataSampleCount)
+        return { };
+    lastVideoFrameMetadataSampleCount = rendered;
+
+    gint width = 0;
+    gint height = 0;
+    if (GstPad *sinkPad = gst_element_get_static_pad(GST_ELEMENT(videoSink.get()), "sink")) {
+        if (GstCaps *caps = gst_pad_get_current_caps(sinkPad)){
+            GstStructure *s = gst_caps_get_structure(caps, 0);
+            gst_structure_get_int(s, "width", &width);
+            gst_structure_get_int(s, "height", &height);
+            gst_caps_unref(caps);
+        }
+        gst_object_unref(sinkPad);
+    }
+
+    const auto now = MonotonicTime::now().secondsSinceEpoch().seconds();
+
+    VideoFrameMetadata metadata;
+    // Convert 90kHz ticks to seconds
+    metadata.mediaTime = MediaTime(pts90kHz, 90000.0).toDouble();
+    metadata.width = width;
+    metadata.height = height;
+    metadata.presentedFrames = rendered;
+
+    // FIXME: presentationTime and expectedDisplayTime might not always have the same value, we should try getting more precise values.
+    metadata.presentationTime = now;
+    metadata.expectedDisplayTime = metadata.presentationTime;
+
+    return metadata;
+}
+
 #undef GST_CAT_DEFAULT
 
 } // namespace WebCore

Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkRialto.h

@@ -38,6 +38,7 @@ class GStreamerHolePunchQuirkRialto final : public GStreamerHolePunchQuirk {
     GstElement* createHolePunchVideoSink(bool, const MediaPlayer*) final;
     bool setHolePunchVideoRectangle(GstElement*, const IntRect&) final;
     bool requiresClockSynchronization() const final { return false; }
+    std::optional<VideoFrameMetadata> videoFrameMetadata(GRefPtr<GstElement>, uint64_t&) final;
 };
 
 } // namespace WebCore

Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkWesteros.cpp

@@ -61,6 +61,49 @@ bool GStreamerHolePunchQuirkWesteros::setHolePunchVideoRectangle(GstElement* vid
     return true;
 }
 
+std::optional<VideoFrameMetadata> GStreamerHolePunchQuirkWesteros::videoFrameMetadata(GRefPtr<GstElement> videoSink, uint64_t& lastVideoFrameMetadataSampleCount)
+{
+    if (UNLIKELY(!gstObjectHasProperty(videoSink.get(), "video_pts")
+        || !gstObjectHasProperty(videoSink.get(), "stats")))
+        return { };
+
+    gint64 pts90kHz = 0;
+    GstStructure* stats = nullptr;
+    gint width = 0, height = 0;
+    g_object_get(videoSink.get(),
+        "video_pts", &pts90kHz,
+        "stats",     &stats,
+        "video_width",     &width,
+        "video_height",    &height,
+        nullptr);
+
+    if (!pts90kHz || !stats)
+        return { };
+
+    guint64 rendered = 0;
+    gst_structure_get_uint64(stats, "rendered", &rendered);
+    gst_structure_free(stats);
+
+    if (rendered == lastVideoFrameMetadataSampleCount)
+        return { };
+    lastVideoFrameMetadataSampleCount = rendered;
+
+    double now = MonotonicTime::now().secondsSinceEpoch().seconds();
+
+    VideoFrameMetadata metadata;
+    // Convert 90kHz ticks to seconds
+    metadata.mediaTime = MediaTime(pts90kHz, 90000.0).toDouble();
+    metadata.width = width;
+    metadata.height = height;
+    metadata.presentedFrames = rendered;
+
+    // FIXME: presentationTime and expectedDisplayTime might not always have the same value, we should try getting more precise values.
+    metadata.presentationTime = now;
+    metadata.expectedDisplayTime = metadata.presentationTime;
+
+    return metadata;
+}
+
 #undef GST_CAT_DEFAULT
 
 } // namespace WebCore

Source/WebCore/platform/gstreamer/GStreamerHolePunchQuirkWesteros.h

@@ -33,6 +33,7 @@ class GStreamerHolePunchQuirkWesteros final : public GStreamerHolePunchQuirk {
     GstElement* createHolePunchVideoSink(bool, const MediaPlayer*) final;
     bool setHolePunchVideoRectangle(GstElement*, const IntRect&) final;
     bool requiresClockSynchronization() const final { return false; }
+    std::optional<VideoFrameMetadata> videoFrameMetadata(GRefPtr<GstElement>, uint64_t&) final;
 };
 
 } // namespace WebCore

Source/WebCore/platform/gstreamer/GStreamerQuirks.cpp

@@ -201,6 +201,14 @@ bool GStreamerQuirksManager::sinksRequireClockSynchronization() const
     return m_holePunchQuirk->requiresClockSynchronization();
 }
 
+std::optional<VideoFrameMetadata> GStreamerQuirksManager::videoFrameMetadata(GRefPtr<GstElement> videoSink, uint64_t& lastVideoFrameMetadataSampleCount)
+{
+    if (!m_holePunchQuirk)
+        return { };
+
+    return m_holePunchQuirk->videoFrameMetadata(videoSink, lastVideoFrameMetadataSampleCount);
+}
+
 void GStreamerQuirksManager::configureElement(GstElement* element, OptionSet<ElementRuntimeCharacteristics>&& characteristics)
 {
     GST_DEBUG("Configuring element %" GST_PTR_FORMAT, element);

Source/WebCore/platform/gstreamer/GStreamerQuirks.h

@@ -24,6 +24,7 @@
 
 #include "GStreamerCommon.h"
 #include "MediaPlayer.h"
+#include "VideoFrameMetadata.h"
 #include <wtf/Forward.h>
 #include <wtf/Noncopyable.h>
 #include <wtf/Nonmovable.h>
@@ -116,6 +117,7 @@ class GStreamerHolePunchQuirk : public GStreamerQuirkBase {
     virtual GstElement* createHolePunchVideoSink(bool, const MediaPlayer*) { return nullptr; }
     virtual bool setHolePunchVideoRectangle(GstElement*, const IntRect&) { return false; }
     virtual bool requiresClockSynchronization() const { return true; }
+    virtual std::optional<VideoFrameMetadata> videoFrameMetadata(GRefPtr<GstElement>, uint64_t&) { return { }; }
 };
 
 class GStreamerQuirksManager : public RefCounted<GStreamerQuirksManager> {
@@ -143,6 +145,7 @@ class GStreamerQuirksManager : public RefCounted<GStreamerQuirksManager> {
     GstElement* createHolePunchVideoSink(bool isLegacyPlaybin, const MediaPlayer*);
     void setHolePunchVideoRectangle(GstElement*, const IntRect&);
     bool sinksRequireClockSynchronization() const;
+    std::optional<VideoFrameMetadata> videoFrameMetadata(GRefPtr<GstElement>, uint64_t&);
 
     void setHolePunchEnabledForTesting(bool);