一、引言
从《音视频入门基础:FLV专题(15)——Video Tag简介》可以知道,未加密的情况下,FLV文件中的一个Video Tag = Tag header + VideoTagHeader + VIDEODATA。本文讲述FFmpeg源码中是怎样解码Video Tag的VideoTagHeader,拿到里面的信息。
二、flv_read_packet函数
从《音视频入门基础:FLV专题(8)——FFmpeg源码中,解码Tag header的实现》可以知道,
FFmpeg源码中使用flv_read_packet函数来读取每个Tag的信息,该函数的前半部分实现了解码Tag header,获取其TagType属性的功能。然后根据TagType属性的值,判断该Tag为音频Tag、视频Tag还是脚本Tag。根据Tag的类型分别执行不同的解码操作:
if (type == FLV_TAG_TYPE_AUDIO) {//...} else if (type == FLV_TAG_TYPE_VIDEO) {//...}else if (type == FLV_TAG_TYPE_META) {//...}else{//...}//...
如果在flv_read_packet函数的前半部分判断出该Tag为Video Tag,flv_read_packet函数中会执行如下逻辑解码Video Tag的VideoTagHeader:
else if (type == FLV_TAG_TYPE_VIDEO) {stream_type = FLV_STREAM_TYPE_VIDEO;flags = avio_r8(s->pb);video_codec_id = flags & FLV_VIDEO_CODECID_MASK;/** Reference Enhancing FLV 2023-03-v1.0.0-B.8* https://github.com/veovera/enhanced-rtmp/blob/main/enhanced-rtmp-v1.pdf* */enhanced_flv = (flags >> 7) & 1;size--;if (enhanced_flv) {video_codec_id = avio_rb32(s->pb);size -= 4;}if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && (flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {int pkt_type = flags & 0x0F;if (pkt_type == PacketTypeMetadata) {int ret = flv_parse_video_color_info(s, st, next);av_log(s, AV_LOG_DEBUG, "enhanced flv parse metadata ret %d and skip\n", ret);}goto skip;} else if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {goto skip;}} //...if (stream_type == FLV_STREAM_TYPE_AUDIO) {
//...}else if (stream_type == FLV_STREAM_TYPE_VIDEO) {int ret = flv_set_video_codec(s, st, video_codec_id, 1);if (ret < 0)return ret;size -= ret;} //...if (st->codecpar->codec_id == AV_CODEC_ID_AAC ||st->codecpar->codec_id == AV_CODEC_ID_H264 ||st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||st->codecpar->codec_id == AV_CODEC_ID_HEVC ||st->codecpar->codec_id == AV_CODEC_ID_AV1 ||st->codecpar->codec_id == AV_CODEC_ID_VP9) {int type = 0;if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO) {type = flags & 0x0F;} else {type = avio_r8(s->pb);size--;}if (size < 0) {ret = AVERROR_INVALIDDATA;goto leave;}if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && flv->meta_color_info_flag) {flv_update_video_color_info(s, st); // update av packet side dataflv->meta_color_info_flag = 0;}if (st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||(st->codecpar->codec_id == AV_CODEC_ID_HEVC && type == PacketTypeCodedFrames)) {// sign extensionint32_t cts = (avio_rb24(s->pb) + 0xff800000) ^ 0xff800000;pts = av_sat_add64(dts, cts);if (cts < 0) { // dts might be wrongif (!flv->wrong_dts)av_log(s, AV_LOG_WARNING,"Negative cts, previous timestamps might be wrong.\n");flv->wrong_dts = 1;} else if (FFABS(dts - pts) > 1000*60*15) {av_log(s, AV_LOG_WARNING,"invalid timestamps %"PRId64" %"PRId64"\n", dts, pts);dts = pts = AV_NOPTS_VALUE;}size -= 3;}
//...}}//...
下面我们分析上述代码块中解码Video Tag的VideoTagHeader的原理。
三、flv_read_packet函数中解码Video Tag的VideoTagHeader的原理
上述代码块中,首先通过avio_r8函数获取VideoTagHeader的第一个字节,也就是Frame Type(占4位) + CodecID(占4位),存贮到局部变量flags中。关于avio_r8函数的用法可以参考:《FFmpeg源码:avio_r8、avio_rl16、avio_rl24、avio_rl32、avio_rl64函数分析》:
flags = avio_r8(s->pb);
宏FLV_VIDEO_CODECID_MASK定义在libavformat/flv.h中:
#define FLV_VIDEO_CODECID_MASK 0x0f
通过下面语句将VideoTagHeader的CodecID属性提取出来,存贮到局部变量video_codec_id中:
video_codec_id = flags & FLV_VIDEO_CODECID_MASK;
下面这部分代码是用来判断文件格式是不是Enhancing FLV的:
/** Reference Enhancing FLV 2023-03-v1.0.0-B.8* https://github.com/veovera/enhanced-rtmp/blob/main/enhanced-rtmp-v1.pdf* */enhanced_flv = (flags >> 7) & 1;size--;if (enhanced_flv) {video_codec_id = avio_rb32(s->pb);size -= 4;}if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && (flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {int pkt_type = flags & 0x0F;if (pkt_type == PacketTypeMetadata) {int ret = flv_parse_video_color_info(s, st, next);av_log(s, AV_LOG_DEBUG, "enhanced flv parse metadata ret %d and skip\n", ret);}goto skip;}
通过VideoTagHeader的Frame Type属性的值来判断该帧是不是视频信息/命令帧。如果是,执行语句:goto skip,不再继续解码VideoTagHeader:
else if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {goto skip;}
由《音视频入门基础:FLV专题(15)——Video Tag简介》可以知道,VideoTagHeader的CodecID属性为编解码器的标识符,表示该Video Tag的视频数据使用的是哪种视频压缩编码方式。通过语句flv_set_video_codec(s, st, video_codec_id, 1)设置st->codecpar->codec_id为CodecID属性对应的视频压缩编码方式:
else if (stream_type == FLV_STREAM_TYPE_VIDEO) {int ret = flv_set_video_codec(s, st, video_codec_id, 1);if (ret < 0)return ret;size -= ret;}
如果文件格式不是Enhancing FLV,通过语句:type = avio_r8(s->pb)读取VideoTagHeader的AVCPacketType属性:
if (st->codecpar->codec_id == AV_CODEC_ID_AAC ||st->codecpar->codec_id == AV_CODEC_ID_H264 ||st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||st->codecpar->codec_id == AV_CODEC_ID_HEVC ||st->codecpar->codec_id == AV_CODEC_ID_AV1 ||st->codecpar->codec_id == AV_CODEC_ID_VP9) {int type = 0;if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO) {type = flags & 0x0F;} else {type = avio_r8(s->pb);size--;}
//...
读取VideoTagHeader的CompositionTime属性,赋值给局部变量cts。通过语句:pts = av_sat_add64(dts, cts),让pts = dts + cts,从而得到pts的值。dts来源于FLV文件的Tag header的Timestamp和TimestampExtended属性,具体可以参考:《音视频入门基础:FLV专题(7)——Tag header简介》和《音视频入门基础:FLV专题(8)——FFmpeg源码中,解码Tag header的实现》。关于av_sat_add64函数的用法可以参考:《FFmpeg源码:av_sat_add64_c、av_sat_sub64_c函数分析》:
if (st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||(st->codecpar->codec_id == AV_CODEC_ID_HEVC && type == PacketTypeCodedFrames)) {// sign extensionint32_t cts = (avio_rb24(s->pb) + 0xff800000) ^ 0xff800000;pts = av_sat_add64(dts, cts);if (cts < 0) { // dts might be wrongif (!flv->wrong_dts)av_log(s, AV_LOG_WARNING,"Negative cts, previous timestamps might be wrong.\n");flv->wrong_dts = 1;} else if (FFABS(dts - pts) > 1000*60*15) {av_log(s, AV_LOG_WARNING,"invalid timestamps %"PRId64" %"PRId64"\n", dts, pts);dts = pts = AV_NOPTS_VALUE;}size -= 3;}