Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use v210 as 10bit pixel format on decklink #21

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

Use v210 as 10bit pixel format on decklink #21

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6edea3f
Use v210 as 10bit pixel format on decklink
niklaspandersson Jun 3, 2024
0229923
simd optimize 16 to 10 bit conversion
niklaspandersson Jun 14, 2024
a845152
avx2 simd for 16 to 10 bit conversion
niklaspandersson Jun 18, 2024
16cd34b
Enfoce alignment of empty const_frames
niklaspandersson Aug 14, 2024
d6c55dd
WIP: custom v210 encoding
niklaspandersson Aug 21, 2024
712024e
[fixup v210] colors
niklaspandersson Sep 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 83 additions & 17 deletions src/modules/decklink/consumer/decklink_consumer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,25 +222,47 @@ class decklink_frame
, public IDeckLinkVideoFrameMetadataExtensions
{
core::video_format_desc format_desc_;
BMDPixelFormat pix_fmt_;
std::shared_ptr<void> data_;
std::atomic<int> ref_count_{0};
int nb_samples_;
const bool hdr_;
core::color_space color_space_;
hdr_meta_configuration hdr_metadata_;
BMDFrameFlags flags_;
BMDPixelFormat pix_fmt_;

public:
decklink_frame(std::shared_ptr<void> data, core::video_format_desc format_desc, int nb_samples, bool hdr, core::color_space color_space, const hdr_meta_configuration& hdr_metadata)
decklink_frame(core::video_format_desc format_desc,
int nb_samples,
bool hdr,
core::color_space color_space,
const hdr_meta_configuration& hdr_metadata,
BMDPixelFormat pix_fmt,
std::shared_ptr<void> data)
: format_desc_(std::move(format_desc))
, pix_fmt_(pix_fmt)
, data_(std::move(data))
, nb_samples_(nb_samples)
, hdr_(hdr)
, color_space_(color_space)
, hdr_metadata_(hdr_metadata)
, flags_(hdr ? bmdFrameFlagDefault | bmdFrameContainsHDRMetadata : bmdFrameFlagDefault)
{
}

decklink_frame(core::video_format_desc format_desc,
int nb_samples,
bool hdr,
core::color_space color_space,
const hdr_meta_configuration& hdr_metadata)
: format_desc_(std::move(format_desc))
, pix_fmt_(get_pixel_format(hdr))
, data_(allocate_frame_data(format_desc, pix_fmt_))
, nb_samples_(nb_samples)
, hdr_(hdr)
, color_space_(color_space)
, hdr_metadata_(hdr_metadata)
, flags_(hdr ? bmdFrameFlagDefault | bmdFrameContainsHDRMetadata : bmdFrameFlagDefault)
{
}

Expand All @@ -255,7 +277,7 @@ class decklink_frame
#else
REFIID iunknown = IID_IUnknown;
#endif
HRESULT result = E_NOINTERFACE;
HRESULT result = E_NOINTERFACE;

if (ppv == nullptr)
return E_INVALIDARG;
Expand Down Expand Up @@ -292,9 +314,12 @@ class decklink_frame

// IDecklinkVideoFrame

long STDMETHODCALLTYPE GetWidth() override { return static_cast<long>(format_desc_.width); }
long STDMETHODCALLTYPE GetHeight() override { return static_cast<long>(format_desc_.height); }
long STDMETHODCALLTYPE GetRowBytes() override { return static_cast<long>(get_row_bytes(format_desc_, hdr_)); }
long STDMETHODCALLTYPE GetWidth() override { return static_cast<long>(format_desc_.width); }
long STDMETHODCALLTYPE GetHeight() override { return static_cast<long>(format_desc_.height); }
long STDMETHODCALLTYPE GetRowBytes() override
{
return static_cast<long>(get_row_bytes(pix_fmt_, format_desc_.width));
}
BMDPixelFormat STDMETHODCALLTYPE GetPixelFormat() override { return pix_fmt_; }
BMDFrameFlags STDMETHODCALLTYPE GetFlags() override { return flags_; }

Expand Down Expand Up @@ -338,7 +363,7 @@ class decklink_frame
HRESULT STDMETHODCALLTYPE GetFloat(BMDDeckLinkFrameMetadataID metadataID, double* value)
{
const auto color_space = (color_space_ == core::color_space::bt2020) ? &REC_2020 : &REC_709;
HRESULT result = S_OK;
HRESULT result = S_OK;

switch (metadataID) {
case bmdDeckLinkFrameMetadataHDRDisplayPrimariesRedX:
Expand Down Expand Up @@ -438,7 +463,7 @@ struct decklink_secondary_port final : public IDeckLinkVideoOutputCallback
const core::video_format_desc decklink_format_desc_;
com_ptr<IDeckLinkDisplayMode> mode_ = get_display_mode(output_,
decklink_format_desc_.format,
get_pixel_format(config_.hdr),
config_.hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA,
bmdSupportedVideoModeDefault,
config_.hdr);

Expand Down Expand Up @@ -555,7 +580,13 @@ struct decklink_secondary_port final : public IDeckLinkVideoOutputCallback
void schedule_next_video(std::shared_ptr<void> image_data, int nb_samples, BMDTimeValue display_time)
{
auto packed_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(
new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples, config_.hdr, core::color_space::bt709, config_.hdr_meta));
new decklink_frame(decklink_format_desc_,
nb_samples,
config_.hdr,
core::color_space::bt709,
config_.hdr_meta,
config_.hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA,
std::move(image_data)));
if (FAILED(output_->ScheduleVideoFrame(get_raw(packed_frame),
display_time,
decklink_format_desc_.duration,
Expand Down Expand Up @@ -624,6 +655,8 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback
bmdSupportedVideoModeDefault,
config_.hdr);

com_ptr<IDeckLinkVideoConversion> video_conversion_;

std::atomic<bool> abort_request_{false};

public:
Expand All @@ -640,6 +673,10 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback
graph_->set_color("buffered-audio", diagnostics::color(0.9f, 0.9f, 0.5f));
graph_->set_color("buffered-video", diagnostics::color(0.2f, 0.9f, 0.9f));

if (config_.hdr) {
video_conversion_ = create_video_converter();
}

if (config.duplex != configuration::duplex_t::default_duplex) {
set_duplex(iface_cast<IDeckLinkAttributes_v10_11>(decklink_),
iface_cast<IDeckLinkConfiguration_v10_11>(decklink_),
Expand Down Expand Up @@ -708,11 +745,12 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback
nb_samples);
}

std::shared_ptr<void> image_data = allocate_frame_data(decklink_format_desc_, config_.hdr);
std::shared_ptr<void> rgb_image_data =
allocate_frame_data(decklink_format_desc_, config_.hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA);

schedule_next_video(image_data, nb_samples, video_scheduled_, config_.hdr_meta.default_color_space);
schedule_next_video(rgb_image_data, nb_samples, video_scheduled_, config_.hdr_meta.default_color_space);
for (auto& context : secondary_port_contexts_) {
context->schedule_next_video(image_data, 0, video_scheduled_);
context->schedule_next_video(rgb_image_data, 0, video_scheduled_);
}

video_scheduled_ += decklink_format_desc_.duration;
Expand Down Expand Up @@ -932,7 +970,8 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback
mode_->GetFieldDominance(),
config_.hdr);

schedule_next_video(image_data, nb_samples, video_display_time, frame1.pixel_format_desc().color_space);
schedule_next_video(
image_data, nb_samples, video_display_time, frame1.pixel_format_desc().color_space);

if (config_.embedded_audio) {
schedule_next_audio(std::move(audio_data), nb_samples);
Expand Down Expand Up @@ -992,12 +1031,39 @@ struct decklink_consumer final : public IDeckLinkVideoOutputCallback
audio_scheduled_ += nb_samples; // TODO - what if there are too many/few samples in this frame?
}

void schedule_next_video(std::shared_ptr<void> image_data, int nb_samples, BMDTimeValue display_time, core::color_space color_space)
void schedule_next_video(std::shared_ptr<void> image_data,
int nb_samples,
BMDTimeValue display_time,
core::color_space color_space)
{
auto fill_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(
new decklink_frame(std::move(image_data), decklink_format_desc_, nb_samples, config_.hdr, color_space, config_.hdr_meta));
auto rgb_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(
new decklink_frame(decklink_format_desc_,
nb_samples,
config_.hdr,
color_space,
config_.hdr_meta,
config_.hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA,
std::move(image_data)));

if (config_.hdr) {
auto yuv_frame = wrap_raw<com_ptr, IDeckLinkVideoFrame>(
new decklink_frame(decklink_format_desc_, nb_samples, config_.hdr, color_space, config_.hdr_meta));

if (FAILED(video_conversion_->ConvertFrame(get_raw(rgb_frame), get_raw(yuv_frame)))) {
CASPAR_LOG(warning) << print() << L" Failed to convert video frame.";
}

if (FAILED(output_->ScheduleVideoFrame(get_raw(yuv_frame),
display_time,
decklink_format_desc_.duration,
decklink_format_desc_.time_scale))) {
CASPAR_LOG(error) << print() << L" Failed to schedule primary video.";
}
return;
}

if (FAILED(output_->ScheduleVideoFrame(
get_raw(fill_frame), display_time, decklink_format_desc_.duration, decklink_format_desc_.time_scale))) {
get_raw(rgb_frame), display_time, decklink_format_desc_.duration, decklink_format_desc_.time_scale))) {
CASPAR_LOG(error) << print() << L" Failed to schedule primary video.";
}
}
Expand Down
64 changes: 50 additions & 14 deletions src/modules/decklink/consumer/frame.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,45 @@

namespace caspar { namespace decklink {

BMDPixelFormat get_pixel_format(bool hdr) { return hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA; }
int get_row_bytes(const core::video_format_desc& format_desc, bool hdr)
BMDPixelFormat get_pixel_format(bool hdr) { return hdr ? bmdFormat10BitYUV : bmdFormat8BitBGRA; }

int get_row_bytes(BMDPixelFormat pix_fmt, int width)
{
return hdr ? ((format_desc.width + 63) / 64) * 256 : format_desc.width * 4;
switch (pix_fmt) {
case bmdFormat10BitYUV:
return ((width + 47) / 48) * 128;
case bmdFormat10BitRGBXLE:
return ((width + 63) / 64) * 256;
default:
break;
}

return width * 4;
}

inline unsigned int pack_pixel(__m128i pixel) {
// Scale down to 10 bit and convert to video range to get a valid
// v210 value after the decklink conversion
// formula: scaled_channel = (src >> 6) * 876 / 1024 + 64;

__m128i bit32 = _mm_unpacklo_epi16(pixel, _mm_setzero_si128()); // unpack 16 bit components to 32 bit
__m128i bit10 = _mm_srli_epi32(bit32, 6); // shift down to 10 bit precision
bit10 = _mm_mullo_epi32(bit10, _mm_set1_epi32(876)); // multiply by 876
bit10 = _mm_srli_epi32(bit10, 10); // divide by 1024
bit10 = _mm_add_epi32(bit10, _mm_set1_epi32(64)); // add 64

// Extract the 10 bit components and save to dest
uint32_t blue = _mm_extract_epi32(bit10, 0);
uint32_t green = _mm_extract_epi32(bit10, 1);
uint32_t red = _mm_extract_epi32(bit10, 2);

return (red << 22) + (green << 12) + (blue << 2);
}

std::shared_ptr<void> allocate_frame_data(const core::video_format_desc& format_desc, bool hdr)
std::shared_ptr<void> allocate_frame_data(const core::video_format_desc& format_desc, BMDPixelFormat pix_fmt)
{
auto alignment = hdr ? 256 : 64;
auto size = hdr ? get_row_bytes(format_desc, hdr) * format_desc.height : format_desc.size;
auto alignment = 256;
auto size = get_row_bytes(pix_fmt, format_desc.width) * format_desc.height;
return create_aligned_buffer(size, alignment);
}

Expand Down Expand Up @@ -72,20 +101,26 @@ void convert_frame(const core::video_format_desc& channel_format_desc,

if (hdr) {
// Pack eight byte R16G16B16A16 pixels as four byte 10bit RGB R10G10B10XX
const int NUM_THREADS = 4;
const int NUM_THREADS = 8;
auto rows_per_thread = decklink_format_desc.height / NUM_THREADS;
size_t byte_count_line = get_row_bytes(decklink_format_desc, hdr);
size_t byte_count_line = get_row_bytes(bmdFormat10BitRGBXLE, decklink_format_desc.width);
tbb::parallel_for(0, NUM_THREADS, [&](int i) {
auto end = (i + 1) * rows_per_thread;
for (int y = firstLine + i * rows_per_thread; y < end; y += decklink_format_desc.field_count) {
auto dest = reinterpret_cast<uint32_t*>(image_data.get()) + (long long)y * byte_count_line / 4;
for (int x = 0; x < decklink_format_desc.width; x += 1) {
__m128i zero = _mm_setzero_si128();
__m128i fac = _mm_set1_epi32(876);
__m128i offset = _mm_set1_epi32(64);

for (int x = 0; x < decklink_format_desc.width; x += 2) {
auto src = reinterpret_cast<const uint16_t*>(
frame.image_data(0).data() + (long long)y * decklink_format_desc.width * 8 + x * 8);
uint16_t blue = src[0] >> 6;
uint16_t green = src[1] >> 6;
uint16_t red = src[2] >> 6;
dest[x] = ((uint32_t)(red) << 22) + ((uint32_t)(green) << 12) + ((uint32_t)(blue) << 2);

// SIMD optimized
// Load two pixels at once to stay on 16-byte aligned memory
__m128i pixels = _mm_load_si128(reinterpret_cast<const __m128i*>(src));
dest[x] = pack_pixel(_mm_unpacklo_epi64(pixels, zero));
dest[x + 1] = pack_pixel(_mm_unpackhi_epi64(pixels, zero));
}
}
});
Expand Down Expand Up @@ -175,7 +210,8 @@ std::shared_ptr<void> convert_frame_for_port(const core::video_format_desc& chan
BMDFieldDominance field_dominance,
bool hdr)
{
std::shared_ptr<void> image_data = allocate_frame_data(decklink_format_desc, hdr);
std::shared_ptr<void> image_data =
allocate_frame_data(decklink_format_desc, hdr ? bmdFormat10BitRGBXLE : bmdFormat8BitBGRA);

if (field_dominance != bmdProgressiveFrame) {
convert_frame(channel_format_desc,
Expand Down
4 changes: 2 additions & 2 deletions src/modules/decklink/consumer/frame.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@
namespace caspar { namespace decklink {

BMDPixelFormat get_pixel_format(bool hdr);
int get_row_bytes(const core::video_format_desc& format_desc, bool hdr);
int get_row_bytes(BMDPixelFormat pix_fmt, int width);

std::shared_ptr<void> allocate_frame_data(const core::video_format_desc& format_desc, bool hdr);
std::shared_ptr<void> allocate_frame_data(const core::video_format_desc& format_desc, BMDPixelFormat pix_fmt);

std::shared_ptr<void> convert_frame_for_port(const core::video_format_desc& channel_format_desc,
const core::video_format_desc& decklink_format_desc,
Expand Down
19 changes: 19 additions & 0 deletions src/modules/decklink/decklink_api.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,15 @@ static com_ptr<IDeckLinkIterator> create_iterator()
return pDecklinkIterator;
}

static com_ptr<IDeckLinkVideoConversion> create_video_converter()
{
CComPtr<IDeckLinkIterator> pVideoConversion_;
if (FAILED(pVideoConversion_.CoCreateInstance(CLSID_CDeckLinkVideoConversion)))
CASPAR_THROW_EXCEPTION(not_supported() << msg_info("Could not create video converter."));

return pVideoConversion_;
}

template <typename I, typename T>
static com_iface_ptr<I> iface_cast(const com_ptr<T>& ptr, bool optional = false)
{
Expand Down Expand Up @@ -164,6 +173,16 @@ static com_ptr<IDeckLinkIterator> create_iterator()
return wrap_raw<com_ptr>(iterator, true);
}

static com_ptr<IDeckLinkVideoConversion> create_video_converter()
{
IDeckLinkVideoConversion* converter = CreateVideoConversionInstance();

if (converter == nullptr)
CASPAR_THROW_EXCEPTION(not_supported() << msg_info("Could not create video converter."));

return wrap_raw<com_ptr>(converter, true);
}

template <typename T>
static REFIID iface_id()
{
Expand Down
Loading