35 #include <TargetConditionals.h> 37 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder 38 # define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder") 40 #ifndef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder 41 # define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder CFSTR("EnableHardwareAcceleratedVideoDecoder") 44 #if !HAVE_KCMVIDEOCODECTYPE_HEVC 48 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12 59 CVPixelBufferRelease(ref->
pixbuf);
136 #define AV_W8(p, v) *(p) = (v) 165 av_assert0(p - vt_extradata == vt_extradata_size);
172 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
180 int i, num_vps = 0, num_sps = 0, num_pps = 0;
189 int vt_extradata_size = 23 + 3 + 3 + 3;
192 #define COUNT_SIZE_PS(T, t) \ 193 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \ 194 if (h->ps.t##ps_list[i]) { \ 195 const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \ 196 vt_extradata_size += 2 + lps->data_size; \ 205 vt_extradata =
av_malloc(vt_extradata_size);
257 AV_W8(p + 15, 0xfc | parallelismType);
286 AV_W8(p + 21, 0 << 6 |
296 #define APPEND_PS(T, t) \ 303 HEVC_NAL_##T##PS & 0x3f); \ 305 AV_WB16(p + 1, num_##t##ps); \ 307 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \ 308 if (h->ps.t##ps_list[i]) { \ 309 const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \ 311 AV_WB16(p, lps->data_size); \ 313 memcpy(p + 2, lps->data, lps->data_size); \ 314 p += 2 + lps->data_size; \ 322 av_assert0(p - vt_extradata == vt_extradata_size);
324 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
356 if (size > 4 && memcmp(vtctx->
sps, buffer + 1, 3) != 0) {
358 memcpy(vtctx->
sps, buffer + 1, 3);
407 CVPixelBufferRelease(vtctx->
frame);
413 #if CONFIG_VIDEOTOOLBOX 431 CVPixelBufferRef
pixbuf = (CVPixelBufferRef)vtctx->
frame;
432 OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
434 int width = CVPixelBufferGetWidth(pixbuf);
435 int height = CVPixelBufferGetHeight(pixbuf);
440 if (!frame->
buf[0] || frame->
data[3]) {
449 CVPixelBufferRelease(ref->
pixbuf);
460 cached_frames->
width != width ||
461 cached_frames->
height != height) {
470 hw_frames->
width = width;
471 hw_frames->
height = height;
496 for (i = 3; i >= 0; i--) {
497 b = (length >> (i * 7)) & 0x7F;
501 bytestream2_put_byteu(pb, b);
505 static CFDataRef videotoolbox_esds_extradata_create(
AVCodecContext *avctx)
519 bytestream2_put_byteu(&pb, 0);
523 bytestream2_put_byteu(&pb, 0x03);
524 videotoolbox_write_mp4_descr_length(&pb, full_size);
526 bytestream2_put_byteu(&pb, 0);
529 bytestream2_put_byteu(&pb, 0x04);
530 videotoolbox_write_mp4_descr_length(&pb, config_size);
531 bytestream2_put_byteu(&pb, 32);
532 bytestream2_put_byteu(&pb, 0x11);
538 bytestream2_put_byteu(&pb, 0x05);
544 bytestream2_put_byteu(&pb, 0x06);
545 bytestream2_put_byteu(&pb, 0x01);
546 bytestream2_put_byteu(&pb, 0x02);
550 data = CFDataCreate(kCFAllocatorDefault, rw_extradata, s);
556 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
561 CMBlockBufferRef block_buf;
562 CMSampleBufferRef sample_buf;
567 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,
578 status = CMSampleBufferCreate(kCFAllocatorDefault,
593 CFRelease(block_buf);
598 static void videotoolbox_decoder_callback(
void *opaque,
599 void *sourceFrameRefCon,
601 VTDecodeInfoFlags
flags,
602 CVImageBufferRef image_buffer,
610 CVPixelBufferRelease(vtctx->
frame);
619 vtctx->
frame = CVPixelBufferRetain(image_buffer);
622 static OSStatus videotoolbox_session_decode_frame(
AVCodecContext *avctx)
625 CMSampleBufferRef sample_buf;
629 sample_buf = videotoolbox_sample_buffer_create(videotoolbox->
cm_fmt_desc,
636 status = VTDecompressionSessionDecodeFrame(videotoolbox->
session,
642 status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->
session);
644 CFRelease(sample_buf);
649 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType
codec_type,
650 CFDictionaryRef decoder_spec,
654 CMFormatDescriptionRef cm_fmt_desc;
657 status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
670 static CFDictionaryRef videotoolbox_buffer_attributes_create(
int width,
674 CFMutableDictionaryRef buffer_attributes;
675 CFMutableDictionaryRef io_surface_properties;
676 CFNumberRef cv_pix_fmt;
680 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
width);
681 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
height);
682 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
684 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
686 &kCFTypeDictionaryKeyCallBacks,
687 &kCFTypeDictionaryValueCallBacks);
688 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
690 &kCFTypeDictionaryKeyCallBacks,
691 &kCFTypeDictionaryValueCallBacks);
694 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
695 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
696 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
697 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
699 CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
701 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
704 CFRelease(io_surface_properties);
705 CFRelease(cv_pix_fmt);
709 return buffer_attributes;
712 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType
codec_type,
715 CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
717 &kCFTypeDictionaryKeyCallBacks,
718 &kCFTypeDictionaryValueCallBacks);
720 CFDictionarySetValue(config_info,
726 CFMutableDictionaryRef avc_info;
729 avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
731 &kCFTypeDictionaryKeyCallBacks,
732 &kCFTypeDictionaryValueCallBacks);
735 case kCMVideoCodecType_MPEG4Video :
737 data = videotoolbox_esds_extradata_create(avctx);
739 CFDictionarySetValue(avc_info, CFSTR(
"esds"), data);
741 case kCMVideoCodecType_H264 :
744 CFDictionarySetValue(avc_info, CFSTR(
"avcC"), data);
749 CFDictionarySetValue(avc_info, CFSTR(
"hvcC"), data);
755 CFDictionarySetValue(config_info,
756 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
770 VTDecompressionOutputCallbackRecord decoder_cb;
771 CFDictionaryRef decoder_spec;
772 CFDictionaryRef buf_attr;
802 decoder_spec = videotoolbox_decoder_config_create(videotoolbox->
cm_codec_type, avctx);
815 CFRelease(decoder_spec);
821 buf_attr = videotoolbox_buffer_attributes_create(avctx->
width,
825 decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
826 decoder_cb.decompressionOutputRefCon = avctx;
828 status = VTDecompressionSessionCreate(
NULL,
836 CFRelease(decoder_spec);
841 case kVTVideoDecoderNotAvailableNowErr:
844 case kVTVideoDecoderUnsupportedDataFormatErr:
847 case kVTCouldNotFindVideoDecoderErr:
850 case kVTVideoDecoderMalfunctionErr:
853 case kVTVideoDecoderBadDataErr:
876 VTDecompressionSessionInvalidate(videotoolbox->
session);
877 CFRelease(videotoolbox->
session);
882 static const char *videotoolbox_error_string(OSStatus status)
885 case kVTVideoDecoderBadDataErr:
887 case kVTVideoDecoderMalfunctionErr:
888 return "decoder malfunction";
889 case kVTInvalidSessionErr:
890 return "invalid session";
909 videotoolbox_stop(avctx);
910 if (videotoolbox_start(avctx) != 0) {
918 status = videotoolbox_session_decode_frame(avctx);
919 if (status != noErr) {
920 if (status == kVTVideoDecoderMalfunctionErr || status == kVTInvalidSessionErr)
922 av_log(avctx,
AV_LOG_ERROR,
"Failed to decode frame (%s, %d)\n", videotoolbox_error_string(status), (
int)status);
931 return videotoolbox_buffer_create(avctx, frame);
939 int ret = videotoolbox_common_end_frame(avctx, frame);
978 int ret = videotoolbox_common_end_frame(avctx, frame);
1004 return videotoolbox_common_end_frame(avctx, frame);
1016 videotoolbox_stop(avctx);
1049 "Either hw_frames_ctx or hw_device_ctx must be set.\n");
1070 hw_frames->
sw_format = videotoolbox_best_pixel_format(avctx);
1095 err = videotoolbox_start(avctx);
1114 frames_ctx->
sw_format = videotoolbox_best_pixel_format(avctx);
1120 .
name =
"h263_videotoolbox",
1125 .start_frame = videotoolbox_mpeg_start_frame,
1126 .decode_slice = videotoolbox_mpeg_decode_slice,
1127 .end_frame = videotoolbox_mpeg_end_frame,
1128 .frame_params = videotoolbox_frame_params,
1129 .init = videotoolbox_common_init,
1135 .
name =
"hevc_videotoolbox",
1140 .start_frame = videotoolbox_hevc_start_frame,
1141 .decode_slice = videotoolbox_hevc_decode_slice,
1142 .decode_params = videotoolbox_hevc_decode_params,
1143 .end_frame = videotoolbox_hevc_end_frame,
1144 .frame_params = videotoolbox_frame_params,
1145 .init = videotoolbox_common_init,
1151 .
name =
"h264_videotoolbox",
1159 .end_frame = videotoolbox_h264_end_frame,
1160 .frame_params = videotoolbox_frame_params,
1161 .init = videotoolbox_common_init,
1167 .
name =
"mpeg1_videotoolbox",
1172 .start_frame = videotoolbox_mpeg_start_frame,
1173 .decode_slice = videotoolbox_mpeg_decode_slice,
1174 .end_frame = videotoolbox_mpeg_end_frame,
1175 .frame_params = videotoolbox_frame_params,
1176 .init = videotoolbox_common_init,
1182 .
name =
"mpeg2_videotoolbox",
1187 .start_frame = videotoolbox_mpeg_start_frame,
1188 .decode_slice = videotoolbox_mpeg_decode_slice,
1189 .end_frame = videotoolbox_mpeg_end_frame,
1190 .frame_params = videotoolbox_frame_params,
1191 .init = videotoolbox_common_init,
1197 .
name =
"mpeg4_videotoolbox",
1202 .start_frame = videotoolbox_mpeg_start_frame,
1203 .decode_slice = videotoolbox_mpeg_decode_slice,
1204 .end_frame = videotoolbox_mpeg_end_frame,
1205 .frame_params = videotoolbox_frame_params,
1206 .init = videotoolbox_common_init,
1219 if (cv_pix_fmt_type == 0) {
1220 cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1240 avctx->
hwaccel_context = vtctx ?: av_videotoolbox_alloc_context_with_pix_fmt(videotoolbox_best_pixel_format(avctx));
1243 return videotoolbox_start(avctx);
1249 videotoolbox_stop(avctx);
int min_spatial_segmentation_idc
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
static enum AVPixelFormat pix_fmt
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it...
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
This structure describes decoded (raw) audio or video data.
int coded_width
Bitstream width / height, may be different from width/height e.g.
int cm_codec_type
CoreMedia codec type that Videotoolbox will use to create the decompression session.
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
hardware decoding through Videotoolbox
static av_always_inline void bytestream2_init_writer(PutByteContext *p, uint8_t *buf, int buf_size)
uint32_t av_map_videotoolbox_format_from_pixfmt(enum AVPixelFormat pix_fmt)
Convert an AVPixelFormat to a VideoToolbox (actually CoreVideo) format.
int width
The allocated dimensions of the frames in this pool.
enum AVMediaType codec_type
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Convenience header that includes libavutil's core.
int is_avc
Used to parse AVC variant of H.264.
enum AVPixelFormat format
The pixel format identifying the underlying HW surface type.
const AVHWAccel ff_h264_videotoolbox_hwaccel
uint8_t entropy_coding_sync_enabled_flag
AVBufferRef * hw_frames_ctx
For hwaccel-format frames, this should be a reference to the AVHWFramesContext describing the frame...
#define av_assert0(cond)
assert() equivalent, that is always enabled.
uint8_t profile_compatibility_flag[32]
AVComponentDescriptor comp[4]
Parameters that describe how pixels are packed.
int ff_attach_decode_data(AVFrame *frame)
void * hwaccel_context
Hardware accelerator context.
AVBufferRef * private_ref
AVBufferRef for internal use by a single libav* library.
An API-specific header for AV_HWDEVICE_TYPE_VIDEOTOOLBOX.
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
#define AV_LOG_VERBOSE
Detailed information.
#define i(width, name, range_min, range_max)
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
uint8_t frame_only_constraint_flag
const AVHWAccel ff_mpeg2_videotoolbox_hwaccel
uint8_t temporal_id_nesting_flag
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
AVBufferRef * av_buffer_create(uint8_t *data, int size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
VTDecompressionSessionRef session
Videotoolbox decompression session object.
int av_hwframe_ctx_init(AVBufferRef *ref)
Finalize the context before use.
uint8_t tiles_enabled_flag
const AVHWAccel ff_hevc_videotoolbox_hwaccel
const char * name
Name of the hardware accelerated codec.
int width
picture width / height.
AVBufferRef * hw_frames_ctx
A reference to the AVHWFramesContext describing the input (for encoding) or output (decoding) frames...
Picture * current_picture_ptr
pointer to the current picture
H.264 / AVC / MPEG-4 part10 codec.
void * av_fast_realloc(void *ptr, unsigned int *size, size_t min_size)
Reallocate the given buffer if it is not large enough, otherwise do nothing.
static av_always_inline unsigned int bytestream2_put_buffer(PutByteContext *p, const uint8_t *src, unsigned int size)
preferred ID for MPEG-1/2 video decoding
int(* post_process)(void *logctx, AVFrame *frame)
The callback to perform some delayed processing on the frame right before it is returned to the calle...
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
#define bytestream2_put_ne24
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
main external API structure.
uint8_t * data
The data buffer.
CMVideoFormatDescriptionRef cm_fmt_desc
CoreMedia Format Description that Videotoolbox will use to create the decompression session...
static int FUNC() pps(CodedBitstreamContext *ctx, RWContext *rw, H264RawPPS *current)
This struct describes a set or pool of "hardware" frames (i.e.
H264Picture * cur_pic_ptr
static int FUNC() sps(CodedBitstreamContext *ctx, RWContext *rw, H264RawSPS *current)
OSType cv_pix_fmt_type
CVPixelBuffer Format Type that Videotoolbox will use for decoded frames.
const AVHWAccel ff_mpeg1_videotoolbox_hwaccel
static int FUNC() vps(CodedBitstreamContext *ctx, RWContext *rw, H265RawVPS *current)
void av_frame_unref(AVFrame *frame)
Unreference all the buffers referenced by frame and reset the frame fields.
#define flags(name, subs,...)
const AVHWAccel ff_mpeg4_videotoolbox_hwaccel
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
AVBufferRef * device_ref
A reference to the parent AVHWDeviceContext.
AVBufferRef * hw_frames_ctx
A reference to a data buffer.
This struct stores per-frame lavc-internal data and is attached to it via private_ref.
VTDecompressionOutputCallback output_callback
The output callback that must be passed to the session.
static int ref[MAX_W *MAX_W]
AVBufferRef * av_hwframe_ctx_alloc(AVBufferRef *device_ref_in)
Allocate an AVHWFramesContext tied to a given device context.
void * hwaccel_priv_data
hwaccel-specific private data
AVBufferRef * av_buffer_ref(AVBufferRef *buf)
Create a new reference to an AVBuffer.
struct AVVideotoolboxContext * vt_ctx
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt)
Convert a VideoToolbox (actually CoreVideo) format to AVPixelFormat.
struct AVCodecInternal * internal
Private context used for internal data.
const AVHWAccel ff_h263_videotoolbox_hwaccel
struct AVBufferRef * cached_hw_frames_ctx
#define bytestream2_put_ne32
uint8_t progressive_source_flag
#define bytestream2_put_ne16
int depth
Number of bits in the component.
AVBufferRef * hw_device_ctx
A reference to the AVHWDeviceContext describing the device which will be used by a hardware encoder/d...
uint8_t non_packed_constraint_flag
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
#define AVERROR_EXTERNAL
Generic error in an external library.
AVPixelFormat
Pixel format.
uint8_t interlaced_source_flag
This struct holds all the information that needs to be passed between the caller and libavcodec for i...
static av_always_inline int bytestream2_size_p(PutByteContext *p)