本笔记记录 FFmpeg HLS Fragmented Mp4 封装过程，主要关注点为 Fragmented Mp4 容器格式。
测试对应的 ffmpeg 命令为:
./ffmpeg -i 2023-06-07-07-40-44.mp4 -c:v copy -c:a copy -hls_segment_type fmp4 -hls_list_size 10 \
-hls_flags delete_segments+append_list -hls_playlist_type event -bsf:a aac_adtstoasc -brand isom \
-loglevel debug -v verbose /tmp/index.m3u8
功能上不涉及编解码，只是将 mp4 文件重新封装为 hls m3u8，其中分片格式使用 fmp4，主要涉及代码为 libavformat 的 hlsenc.c，基础数据结构如下：
const FFOutputFormat ff_hls_muxer = {
.p.name = "hls",
.p.long_name = NULL_IF_CONFIG_SMALL("Apple HTTP Live Streaming"),
.p.extensions = "m3u8",
.p.audio_codec = AV_CODEC_ID_AAC,
.p.video_codec = AV_CODEC_ID_H264,
.p.subtitle_codec = AV_CODEC_ID_WEBVTT,
.p.flags = AVFMT_NOFILE | AVFMT_GLOBALHEADER | AVFMT_ALLOW_FLUSH | AVFMT_NODIMENSIONS,
.p.priv_class = &hls_class,
.priv_data_size = sizeof(HLSContext),
.init = hls_init,
.write_header = hls_write_header,
.write_packet = hls_write_packet,
.write_trailer = hls_write_trailer,
.deinit = hls_deinit,
};
typedef struct HLSContext {
const AVClass *class; // Class for private options.
int64_t start_sequence;
uint32_t start_sequence_source_type; // enum StartSequenceSourceType
int64_t time; // Set by a private option.
int64_t init_time; // Set by a private option.
int max_nb_segments; // Set by a private option.
int hls_delete_threshold; // Set by a private option.
uint32_t flags; // enum HLSFlags
uint32_t pl_type; // enum PlaylistType
char *segment_filename;
char *fmp4_init_filename;
int segment_type;
int resend_init_file; ///< resend init file into disk after refresh m3u8
int use_localtime; ///< flag to expand filename with localtime
int use_localtime_mkdir;///< flag to mkdir dirname in timebased filename
int allowcache;
int64_t recording_time;
int64_t max_seg_size; // every segment file max size
char *baseurl;
char *vtt_format_options_str;
char *subtitle_filename;
AVDictionary *format_options;
int encrypt;
char *key;
char *key_url;
char *iv;
char *key_basename;
int encrypt_started;
char *key_info_file;
char key_file[LINE_BUFFER_SIZE + 1];
char key_uri[LINE_BUFFER_SIZE + 1];
char key_string[KEYSIZE*2 + 1];
char iv_string[KEYSIZE*2 + 1];
AVDictionary *vtt_format_options;
char *method;
char *user_agent;
VariantStream *var_streams;
unsigned int nb_varstreams;
ClosedCaptionsStream *cc_streams;
unsigned int nb_ccstreams;
int master_m3u8_created; /* status of master play-list creation */
char *master_m3u8_url; /* URL of the master m3u8 file */
int version; /* HLS version */
char *var_stream_map; /* user specified variant stream map string */
char *cc_stream_map; /* user specified closed caption streams map string */
char *master_pl_name;
unsigned int master_publish_rate;
int http_persistent;
AVIOContext *m3u8_out;
AVIOContext *sub_m3u8_out;
AVIOContext *http_delete;
int64_t timeout;
int ignore_io_errors;
char *headers;
int has_default_key; /* has DEFAULT field of var_stream_map */
int has_video_m3u8; /* has video stream m3u8 list */
} HLSContext;
#define OFFSET(x) offsetof(HLSContext, x)
#define E AV_OPT_FLAG_ENCODING_PARAM
static const AVOption options[] = {
{"start_number", "set first number in the sequence", OFFSET(start_sequence),AV_OPT_TYPE_INT64, {.i64 = 0}, 0, INT64_MAX, E},
{"hls_time", "set segment length", OFFSET(time), AV_OPT_TYPE_DURATION, {.i64 = 2000000}, 0, INT64_MAX, E},
{"hls_init_time", "set segment length at init list", OFFSET(init_time), AV_OPT_TYPE_DURATION, {.i64 = 0}, 0, INT64_MAX, E},
{"hls_list_size", "set maximum number of playlist entries", OFFSET(max_nb_segments), AV_OPT_TYPE_INT, {.i64 = 5}, 0, INT_MAX, E},
{"hls_delete_threshold", "set number of unreferenced segments to keep before deleting", OFFSET(hls_delete_threshold), AV_OPT_TYPE_INT, {.i64 = 1}, 1, INT_MAX, E},
{"hls_vtt_options","set hls vtt list of options for the container format used for hls", OFFSET(vtt_format_options_str), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"hls_allow_cache", "explicitly set whether the client MAY (1) or MUST NOT (0) cache media segments", OFFSET(allowcache), AV_OPT_TYPE_INT, {.i64 = -1}, INT_MIN, INT_MAX, E},
{"hls_base_url", "url to prepend to each playlist entry", OFFSET(baseurl), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"hls_segment_filename", "filename template for segment files", OFFSET(segment_filename), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"hls_segment_options","set segments files format options of hls", OFFSET(format_options), AV_OPT_TYPE_DICT, {.str = NULL}, 0, 0, E},
{"hls_segment_size", "maximum size per segment file, (in bytes)", OFFSET(max_seg_size), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, E},
{"hls_key_info_file", "file with key URI and key file path", OFFSET(key_info_file), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"hls_enc", "enable AES128 encryption support", OFFSET(encrypt), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, E},
{"hls_enc_key", "hex-coded 16 byte key to encrypt the segments", OFFSET(key), AV_OPT_TYPE_STRING, .flags = E},
{"hls_enc_key_url", "url to access the key to decrypt the segments", OFFSET(key_url), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"hls_enc_iv", "hex-coded 16 byte initialization vector", OFFSET(iv), AV_OPT_TYPE_STRING, .flags = E},
{"hls_subtitle_path", "set path of hls subtitles", OFFSET(subtitle_filename), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"hls_segment_type", "set hls segment files type", OFFSET(segment_type), AV_OPT_TYPE_INT, {.i64 = SEGMENT_TYPE_MPEGTS }, 0, SEGMENT_TYPE_FMP4, E, "segment_type"},
{"mpegts", "make segment file to mpegts files in m3u8", 0, AV_OPT_TYPE_CONST, {.i64 = SEGMENT_TYPE_MPEGTS }, 0, UINT_MAX, E, "segment_type"},
{"fmp4", "make segment file to fragment mp4 files in m3u8", 0, AV_OPT_TYPE_CONST, {.i64 = SEGMENT_TYPE_FMP4 }, 0, UINT_MAX, E, "segment_type"},
{"hls_fmp4_init_filename", "set fragment mp4 file init filename", OFFSET(fmp4_init_filename), AV_OPT_TYPE_STRING, {.str = "init.mp4"}, 0, 0, E},
{"hls_RtspServerfmp4_init_resend", "resend fragment mp4 init file after refresh m3u8 every time", OFFSET(resend_init_file), AV_OPT_TYPE_BOOL, {.i64 = 0 }, 0, 1, E },
{"hls_flags", "set flags affecting HLS playlist and media file generation", OFFSET(flags), AV_OPT_TYPE_FLAGS, {.i64 = 0 }, 0, UINT_MAX, E, "flags"},
{"single_file", "generate a single media file indexed with byte ranges", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SINGLE_FILE }, 0, UINT_MAX, E, "flags"},
{"temp_file", "write segment and playlist to temporary file and rename when complete", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_TEMP_FILE }, 0, UINT_MAX, E, "flags"},
{"delete_segments", "delete segment files that are no longer part of the playlist", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_DELETE_SEGMENTS }, 0, UINT_MAX, E, "flags"},
{"round_durations", "round durations in m3u8 to whole numbers", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_ROUND_DURATIONS }, 0, UINT_MAX, E, "flags"},
{"discont_start", "start the playlist with a discontinuity tag", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_DISCONT_START }, 0, UINT_MAX, E, "flags"},
{"omit_endlist", "Do not append an endlist when ending stream", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_OMIT_ENDLIST }, 0, UINT_MAX, E, "flags"},
{"split_by_time", "split the hls segment by time which user set by hls_time", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SPLIT_BY_TIME }, 0, UINT_MAX, E, "flags"},
{"append_list", "append the new segments into old hls segment list", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_APPEND_LIST }, 0, UINT_MAX, E, "flags"},
{"program_date_time", "add EXT-X-PROGRAM-DATE-TIME", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_PROGRAM_DATE_TIME }, 0, UINT_MAX, E, "flags"},
{"second_level_segment_index", "include segment index in segment filenames when use_localtime", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SECOND_LEVEL_SEGMENT_INDEX }, 0, UINT_MAX, E, "flags"},
{"second_level_segment_duration", "include segment duration in segment filenames when use_localtime", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SECOND_LEVEL_SEGMENT_DURATION }, 0, UINT_MAX, E, "flags"},
{"second_level_segment_size", "include segment size in segment filenames when use_localtime", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_SECOND_LEVEL_SEGMENT_SIZE }, 0, UINT_MAX, E, "flags"},
{"periodic_rekey", "reload keyinfo file periodically for re-keying", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_PERIODIC_REKEY }, 0, UINT_MAX, E, "flags"},
{"independent_segments", "add EXT-X-INDEPENDENT-SEGMENTS, whenever applicable", 0, AV_OPT_TYPE_CONST, { .i64 = HLS_INDEPENDENT_SEGMENTS }, 0, UINT_MAX, E, "flags"},
{"iframes_only", "add EXT-X-I-FRAMES-ONLY, whenever applicable", 0, AV_OPT_TYPE_CONST, { .i64 = HLS_I_FRAMES_ONLY }, 0, UINT_MAX, E, "flags"},
{"strftime", "set filename expansion with strftime at segment creation", OFFSET(use_localtime), AV_OPT_TYPE_BOOL, {.i64 = 0 }, 0, 1, E },
{"strftime_mkdir", "create last directory component in strftime-generated filename", OFFSET(use_localtime_mkdir), AV_OPT_TYPE_BOOL, {.i64 = 0 }, 0, 1, E },
{"hls_playlist_type", "set the HLS playlist type", OFFSET(pl_type), AV_OPT_TYPE_INT, {.i64 = PLAYLIST_TYPE_NONE }, 0, PLAYLIST_TYPE_NB-1, E, "pl_type" },
{"event", "EVENT playlist", 0, AV_OPT_TYPE_CONST, {.i64 = PLAYLIST_TYPE_EVENT }, INT_MIN, INT_MAX, E, "pl_type" },
{"vod", "VOD playlist", 0, AV_OPT_TYPE_CONST, {.i64 = PLAYLIST_TYPE_VOD }, INT_MIN, INT_MAX, E, "pl_type" },
{"method", "set the HTTP method(default: PUT)", OFFSET(method), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"hls_start_number_source", "set source of first number in sequence", OFFSET(start_sequence_source_type), AV_OPT_TYPE_INT, {.i64 = HLS_START_SEQUENCE_AS_START_NUMBER }, 0, HLS_START_SEQUENCE_LAST-1, E, "start_sequence_source_type" },
{"generic", "start_number value (default)", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_START_SEQUENCE_AS_START_NUMBER }, INT_MIN, INT_MAX, E, "start_sequence_source_type" },
{"epoch", "seconds since epoch", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_START_SEQUENCE_AS_SECONDS_SINCE_EPOCH }, INT_MIN, INT_MAX, E, "start_sequence_source_type" },
{"epoch_us", "microseconds since epoch", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_START_SEQUENCE_AS_MICROSECONDS_SINCE_EPOCH }, INT_MIN, INT_MAX, E, "start_sequence_source_type" },
{"datetime", "current datetime as YYYYMMDDhhmmss", 0, AV_OPT_TYPE_CONST, {.i64 = HLS_START_SEQUENCE_AS_FORMATTED_DATETIME }, INT_MIN, INT_MAX, E, "start_sequence_source_type" },
{"http_user_agent", "override User-Agent field in HTTP header", OFFSET(user_agent), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"var_stream_map", "Variant stream map string", OFFSET(var_stream_map), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"cc_stream_map", "Closed captions stream map string", OFFSET(cc_stream_map), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"master_pl_name", "Create HLS master playlist with this name", OFFSET(master_pl_name), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, E},
{"master_pl_publish_rate", "Publish master play list every after this many segment intervals", OFFSET(master_publish_rate), AV_OPT_TYPE_INT, {.i64 = 0}, 0, UINT_MAX, E},
{"http_persistent", "Use persistent HTTP connections", OFFSET(http_persistent), AV_OPT_TYPE_BOOL, {.i64 = 0 }, 0, 1, E },
{"timeout", "set timeout for socket I/O operations", OFFSET(timeout), AV_OPT_TYPE_DURATION, { .i64 = -1 }, -1, INT_MAX, .flags = E },
{"ignore_io_errors", "Ignore IO errors for stable long-duration runs with network output", OFFSET(ignore_io_errors), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, E },
{"headers", "set custom HTTP headers, can override built in default headers", OFFSET(headers), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, E },
{ NULL },
};
static const AVClass hls_class = {
.class_name = "hls muxer",
.item_name = av_default_item_name,
.option = options,
.version = LIBAVUTIL_VERSION_INT,
};
ff_hls_muxer 调用的生命周期如下:
其中重点关注 hls_write_packet 函数，其调用流程如下：
- main 函数调用 ffmpeg_parse_options，通过 of_open 申请内存，初始化 Muxer，并放入全局 output_files 数组
- of_open 中较关键为 avformat_alloc_output_context2 ，根据输出格式申请 AVFormatContext
- ffmpeg_mux.c 初始化后启动 muxer_thread，唯一参数为 Muxer，
- muxer_thread 轮询 mux->tq 读取帧信息 AVPacket
- ffmpeg_mux.c write_packet 修正时间相关数据
- mux.c write_packets_common, 如果该 bitstream 没有 filter 则通过 write_packet_common 处理
- mux.c write_packet_common 处理帧时长，注意此时参数 AVFormatContext 为 mux->fc，即 hls
- mux.c write_packet 修正相关数据
- hlsenc.c hls_write_packet
hls_write_packet 函数实现的功能如下：
- 找出 AVPacket 对应的 VariantStream 与 AVFormatContext
- 修正 end_pts 与 start_pts 时间参数
- 检查是否需要切分新 Fragment，譬如是否设置 split_by_time 参数，当前帧是否为关键帧
- 处理 ref_pkt （暂时未知作用）
- 如果通过 split_by_time 参数指定分割时间，并且为I帧，则进行分割操作, 通过 av_write_frame 写入 NULL pkt，触发 movenc.c mov_write_packet 函数调用 mov_flush_fragment 完成
- 如果不需要分片，则将 AVPacket 发送到 ff_write_chained 处理，注意此时的 AVFormatContext 已经转换为 s->priv_data->var_streams[i]->avf，即 mp4
- ff_write_chained 修正 pts 后发送到 av_write_frame
- 重新进入 mux.c write_packets_common，只不过此时 pkt 为 mp4 格式，会发送到 movenc.c 的 mov_write_packet 函数进行处理
mov_write_packet 为 Mp4 帧封装函数，实现的功能如下：
- 如果接收到空 AVPacket，则调用 mov_flush_fragment 进行 Fragmented Mp4 封装，否则调用 mov_write_single_packet（不考虑与我们场景不相干分支）
- mov_write_single_packet 先修正 AVPacket 数据，获取 mov 与 trk 相关参数，判断是否需要 mov_auto_flush_fragment，然后调用 ff_mov_write_packet；
- ff_mov_write_packet 写入 mdat，然后将帧数据 MOVIentry 添加到 trk->cluster，后续写入 fMp4 相关数据 Box 时会大量依赖 trk->cluster
mov_flush_fragment 为 Fragmented Mp4 文件封装函数，实现功能如下：
- 计算、修正 track 的时长与 end_pts
- 写 moov box
- 写 moof box