【音视频】FFmpeg过滤器框架分析
ffmpeg的filter⽤起来是和Gstreamer的plugin是⼀样的概念，通过avfilter_link，将各个创建好的filter按⾃⼰想要的次序链接到⼀起，然后avfilter_graph_config之后，就可以正常使⽤。
⽐较常⽤的滤镜有：scale、trim、overlay、rotate、movie、yadif。scale 滤镜⽤于缩放，trim 滤镜⽤于帧级剪切，overlay 滤镜⽤于视频叠加，rotate 滤镜实现旋转，movie 滤镜可以加载第三⽅的视频，yadif 滤镜可以去隔⾏。
1 主要结构体和API介绍
AVFilterGraph-对filters系统的整体管理
重点
struct AVFilterGraph
{
AVFilterContext **filters;
unsigned nb_filters;
}
完整结构体
// 对filters系统的整体管理
typedef struct AVFilterGraph {const AVClass *av_class;AVFilterContext **filters;unsigned nb_filters;char *scale_sws_opts; ///< sws options to use for the auto-inser
ted scale filters
#if FF_API_LAVR_OPTSattribute_deprecated char *resample_lavr_opts; ///< libavresam
ple options to use for the auto-inserted resample filters
#endif/*** Type of multithreading allowed for filters in this graph. A c
ombination* of AVFILTER_THREAD_* flags.** May be set by the caller at any point, the setting will apply
to all* filters initialized after that. The default is allowing every
thing.** When a filter in this graph is initialized, this field is com
bined using* bit AND with AVFilterContext.thread_type to get the final mas
k used for* determining allowed threading types. I.e. a threading type ne
eds to be* set in both to be allowed.*/int thread_type;/*** Maximum number of threads used by filters in this graph. May
be set by* the caller before adding any filters to the filtergraph. Zero
(the* default) means that the number of threads is determined autom
atically.*/int nb_threads;/*** Opaque object for libavfilter internal use.*/AVFilterGraphInternal *internal;/*** Opaque user data. May be set by the caller to an arbitrary va
lue, e.g. to* be used from callbacks like @ref AVFilterGraph.execute.* Libavfilter will not touch this field in any way.*/void *opaque;/*** This callback may be set by the caller immediately after allo
cating the* graph and before adding any filters to it, to provide a custo
m* multithreading implementation.** If set, filters with slice threading capability will call thi
s callback* to execute multiple jobs in parallel.** If this field is left unset, libavfilter will use its interna
l* implementation, which may or may not be multithreaded dependi
ng on the* platform and build options.*/avfilter_execute_func *execute;char *aresample_swr_opts; ///< swr options to use for the auto-i
nserted aresample filters, Access ONLY through AVOptions/*** Private fields** The following fields are for internal use only.* Their type, offset, number and semantic can change without no
tice.*/AVFilterLink **sink_links;int sink_links_count;unsigned disable_auto_convert;
} AVFilterGraph;
AVFilter-定义filter本身的能⼒
重点
const char *name; // overlay
const AVFilterPad *inputs;
const AVFilterPad *outputs;
比如
AVFilter ff_vf_overlay = {.name = "overlay",.description = NULL_IF_CONFIG_SMALL("Overlay a video source on top of the input."),.preinit = overlay_framesync_preinit,.init = init,.uninit = uninit,.priv_size = sizeof(OverlayContext),.priv_class = &overlay_class,.query_formats = query_formats,.activate = activate,.process_command = process_command,.inputs = avfilter_vf_overlay_inputs,.outputs = avfilter_vf_overlay_outputs,.flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL |AVFILTER_FLAG_SLICE_THREADS,
};
定义filter本身的能⼒，拥有的pads，回调函数接⼝定义
*/
typedef struct AVFilter {/*** Filter name. Must be non-NULL and unique among filters.*/const char *name;/*** A description of the filter. May be NULL.** You should use the NULL_IF_CONFIG_SMALL() macro to define it.*/const char *description;/*** List of inputs, terminated by a zeroed element.** NULL if there are no (static) inputs. Instances of filters with* AVFILTER_FLAG_DYNAMIC_INPUTS set may have more inputs than present in* this list.*/const AVFilterPad *inputs;/*** List of outputs, terminated by a zeroed element.** NULL if there are no (static) outputs. Instances of filters with* AVFILTER_FLAG_DYNAMIC_OUTPUTS set may have more outputs than present in* this list.*/const AVFilterPad *outputs;/*** A class for the private data, used to declare filter private AVOptions.* This field is NULL for filters that do not declare any options.** If this field is non-NULL, the first member of the filter private data* must be a pointer to AVClass, which will be set by libavfilter generic* code to this class.*/const AVClass *priv_class;/*** A combination of AVFILTER_FLAG_**/int flags;/**************************************************************** All fields below this line are not part of the public API. They* may not be used outside of libavfilter and can be changed and* removed at will.* New public fields should be added right above.***************************************************************//*** Filter pre-initialization function** This callback will be called immediately after the filter context is* allocated, to allow allocating and initing sub-objects.** If this callback is not NULL, the uninit callback will be called on* allocation failure.** @return 0 on success,* AVERROR code on failure (but the code will be* dropped and treated as ENOMEM by the calling code)*/int (*preinit)(AVFilterContext *ctx);/*** Filter initialization function.** This callback will be called only once during the filter lifetime, after* all the options have been set, but before links between filters are* established and format negotiation is done.** Basic filter initialization should be done here. Filters with dynamic* inputs and/or outputs should create those inputs/outputs here based on* provided options. No more changes to this filter's inputs/outputs can be* done after this callback.** This callback must not assume that the filter links exist or frame* parameters are known.** @ref AVFilter.uninit "uninit" is guaranteed to be called even if* initialization fails, so this callback does not have to clean up on* failure.** @return 0 on success, a negative AVERROR on failure*/int (*init)(AVFilterContext *ctx);/*** Should be set instead of @ref AVFilter.init "init" by the filters that* want to pass a dictionary of AVOptions to nested contexts that are* allocated during init.** On return, the options dict should be freed and replaced with one that* contains all the options which could not be processed by this filter (or* with NULL if all the options were processed).** Otherwise the semantics is the same as for @ref AVFilter.init "init".*/int (*init_dict)(AVFilterContext *ctx, AVDictionary **options);/*** Filter uninitialization function.** Called only once right before the filter is freed. Should deallocate any* memory held by the filter, release any buffer references, etc. It does* not need to deallocate the AVFilterContext.priv memory itself.** This callback may be called even if @ref AVFilter.init "init" was not* called or failed, so it must be prepared to handle such a situation.*/void (*uninit)(AVFilterContext *ctx);/*** Query formats supported by the filter on its inputs and outputs.** This callback is called after the filter is initialized (so the inputs* and outputs are fixed), shortly before the format negotiation. This* callback may be called more than once.** This callback must set AVFilterLink.out_formats on every input link and* AVFilterLink.in_formats on every output link to a list of pixel/sample* formats that the filter supports on that link. For audio links, this* filter must also set @ref AVFilterLink.in_samplerates "in_samplerates" /* @ref AVFilterLink.out_samplerates "out_samplerates" and* @ref AVFilterLink.in_channel_layouts "in_channel_layouts" /* @ref AVFilterLink.out_channel_layouts "out_channel_layouts" analogously.** This callback may be NULL for filters with one input, in which case* libavfilter assumes that it supports all input formats and reserves* them on output.** @return zero on success, a negative value corresponding to an* AVERROR code otherwise*/int (*query_formats)(AVFilterContext *);int priv_size; ///< size of private data to allocate for the filterint flags_internal; ///< Additional flags for avfilter internal use only./*** Used by the filter registration system. Must not be touched by any other* code.*/struct AVFilter *next;/*** Make the filter instance process a command.** @param cmd the command to process, for handling simplicity all commands must be alphanumeric only* @param arg the argument for the command* @param res a buffer with size res_size where the filter(s) can return a response. This must not change when the command is* not supported.* @param flags if AVFILTER_CMD_FLAG_FAST is set and the command would be* time consuming then a filter should treat it like an unsupported command** @returns >=0 on success otherwise an error code.* AVERROR(ENOSYS) on unsupported commands*/int (*process_command)(AVFilterContext *, const char *cmd, const char *arg, char *res, int res_len, int flags);/*** Filter initialization function, alternative to the init()* callback. Args contains the user-supplied parameters, opaque is* used for providing binary data.*/int (*init_opaque)(AVFilterContext *ctx, void *opaque);/*** Filter activation function.** Called when any processing is needed from the filter, instead of any* filter_frame and request_frame on pads.** The function must examine inlinks and outlinks and perform a single* step of processing. If there is nothing to do, the function must do* nothing and not return an error. If more steps are or may be* possible, it must use ff_filter_set_ready() to schedule another* activation.*/int (*activate)(AVFilterContext *ctx);
} AVFilter;
AVFilterContext-filter实例，管理filter与外部的联系
filter实例，管理filter与外部的联系
重点
struct AVFilterContext
{const AVFilter *filter;char *name;AVFilterPad *input_pads;AVFilterLink **inputs;unsigned nb_inputsAVFilterPad *output_pads;AVFilterLink **outputs;unsigned nb_outputs;struct AVFilterGraph *graph; // 从属于哪个AVFilterGraph
}
完整结构体
/** An instance of a filter */
struct AVFilterContext {const AVClass *av_class; ///< needed for av_log() and filters common optionsconst AVFilter *filter; ///< the AVFilter of which this is an instancechar *name; ///< name of this filter instanceAVFilterPad *input_pads; ///< array of input padsAVFilterContext-filter实例，管理filter与外部的联系AVFilterLink **inputs; ///< array of pointers to input linksunsigned nb_inputs; ///< number of input padsAVFilterPad *output_pads; ///< array of output padsAVFilterLink **outputs; ///< array of pointers to output linksunsigned nb_outputs; ///< number of output padsvoid *priv; ///< private data for use by the filterstruct AVFilterGraph *graph; ///< filtergraph this filter belongs to/*** Type of multithreading being allowed/used. A combination of* AVFILTER_THREAD_* flags.** May be set by the caller before initializing the filter to forbid some* or all kinds of multithreading for this filter. The default is allowing* everything.** When the filter is initialized, this field is combined using bit AND with* AVFilterGraph.thread_type to get the final mask used for determining* allowed threading types. I.e. a threading type needs to be set in both* to be allowed.** After the filter is initialized, libavfilter sets this field to the* threading type that is actually used (0 for no multithreading).*/int thread_type;/*** An opaque struct for libavfilter internal use.*/AVFilterInternal *internal;struct AVFilterCommand *command_queue;char *enable_str; ///< enable expression stringvoid *enable; ///< parsed expression (AVExpr*)double *var_values; ///< variable values for the enable expressionint is_disabled; ///< the enabled state from the last expression evaluation/*** For filters which will create hardware frames, sets the device the* filter should create them in. All other filters will ignore this field:* in particular, a filter which consumes or processes hardware frames will* instead use the hw_frames_ctx field in AVFilterLink to carry the* hardware context information.*/AVBufferRef *hw_device_ctx;/*** Max number of threads allowed in this filter instance.* If <= 0, its value is ignored.* Overrides global number of threads set per filter graph.*/int nb_threads;/*** Ready status of the filter.* A non-0 value means that the filter needs activating;* a higher value suggests a more urgent activation.*/unsigned ready;/*** Sets the number of extra hardware frames which the filter will* allocate on its output links for use in following filters or by* the caller.** Some hardware filters require all frames that they will use for* output to be defined in advance before filtering starts. For such* filters, any hardware frame pools used for output must therefore be* of fixed size. The extra frames set here are on top of any number* that the filter needs internally in order to operate normally.** This field must be set before the graph containing this filter is* configured.*/int extra_hw_frames;
};
AVFilterLink-定义两个filters之间的联接
重点
struct AVFilterLink
{AVFilterContext *src;AVFilterPad *srcpad;AVFilterContext *dst;AVFilterPad *dstpad;struct AVFilterGraph *graph;
};
完整结构体
/*** A link between two filters. This contains pointers to the source and* destination filters between which this link exists, and the indexes of* the pads involved. In addition, this link also contains the parameters* which have been negotiated and agreed upon between the filter, such as* image dimensions, format, etc.** Applications must not normally access the link structure directly.* Use the buffersrc and buffersink API instead.* In the future, access to the header may be reserved for filters* implementation.*/
struct AVFilterLink {AVFilterContext *src; ///< source filterAVFilterPad *srcpad; ///< output pad on the source filterAVFilterContext *dst; ///< dest filterAVFilterPad *dstpad; ///< input pad on the dest filterenum AVMediaType type; ///< filter media type/* These parameters apply only to video */int w; ///< agreed upon image widthint h; ///< agreed upon image heightAVRational sample_aspect_ratio; ///< agreed upon sample aspect ratio/* These parameters apply only to audio */uint64_t channel_layout; ///< channel layout of current buffer (see libavutil/channel_layout.h)int sample_rate; ///< samples per secondint format; ///< agreed upon media format/*** Define the time base used by the PTS of the frames/samples* which will pass through this link.* During the configuration stage, each filter is supposed to* change only the output timebase, while the timebase of the* input link is assumed to be an unchangeable property.*/AVRational time_base;/**************************************************************** All fields below this line are not part of the public API. They* may not be used outside of libavfilter and can be changed and* removed at will.* New public fields should be added right above.***************************************************************//*** Lists of formats and channel layouts supported by the input* and output filters respectively. These lists are used for negotiating the format* to actually be used, which will be loaded into the format and* channel_layout members, above, when chosen.**/AVFilterFormats *in_formats;AVFilterFormats *out_formats;/*** Lists of channel layouts and sample rates used for automatic* negotiation.*/AVFilterFormats *in_samplerates;AVFilterFormats *out_samplerates;struct AVFilterChannelLayouts *in_channel_layouts;struct AVFilterChannelLayouts *out_channel_layouts;/*** Audio only, the destination filter sets this to a non-zero value to* request that buffers with the given number of samples should be sent to* it. AVFilterPad.needs_fifo must also be set on the corresponding input* pad.* Last buffer before EOF will be padded with silence.*/int request_samples;/** stage of the initialization of the link properties (dimensions, etc) */enum {AVLINK_UNINIT = 0, ///< not startedAVLINK_STARTINIT, ///< started, but incompleteAVLINK_INIT ///< complete} init_state;/*** Graph the filter belongs to.*/struct AVFilterGraph *graph;/*** Current timestamp of the link, as defined by the most recent* frame(s), in link time_base units.*/int64_t current_pts;/*** Current timestamp of the link, as defined by the most recent* frame(s), in AV_TIME_BASE units.*/int64_t current_pts_us;/*** Index in the age array.*/int age_index;/*** Frame rate of the stream on the link, or 1/0 if unknown or variable;* if left to 0/0, will be automatically copied from the first input* of the source filter if it exists.** Sources should set it to the best estimation of the real frame rate.* If the source frame rate is unknown or variable, set this to 1/0.* Filters should update it if necessary depending on their function.* Sinks can use it to set a default output frame rate.* It is similar to the r_frame_rate field in AVStream.*/AVRational frame_rate;/*** Buffer partially filled with samples to achieve a fixed/minimum size.*/AVFrame *partial_buf;/*** Size of the partial buffer to allocate.* Must be between min_samples and max_samples.*/int partial_buf_size;/*** Minimum number of samples to filter at once. If filter_frame() is* called with fewer samples, it will accumulate them in partial_buf.* This field and the related ones must not be changed after filtering* has started.* If 0, all related fields are ignored.*/int min_samples;/*** Maximum number of samples to filter at once. If filter_frame() is* called with more samples, it will split them.*/int max_samples;/*** Number of channels.*/int channels;/*** Link processing flags.*/unsigned flags;/*** Number of past frames sent through the link.*/int64_t frame_count_in, frame_count_out;/*** A pointer to a FFFramePool struct.*/void *frame_pool;/*** True if a frame is currently wanted on the output of this filter.* Set when ff_request_frame() is called by the output,* cleared when a frame is filtered.*/int frame_wanted_out;/*** For hwaccel pixel formats, this should be a reference to the* AVHWFramesContext describing the frames.*/AVBufferRef *hw_frames_ctx;#ifndef FF_INTERNAL_FIELDS/*** Internal structure members.* The fields below this limit are internal for libavfilter's use* and must in no way be accessed by applications.*/char reserved[0xF000];#else /* FF_INTERNAL_FIELDS *//*** Queue of frames waiting to be filtered.*/FFFrameQueue fifo;/*** If set, the source filter can not generate a frame as is.* The goal is to avoid repeatedly calling the request_frame() method on* the same link.*/int frame_blocked_in;/*** Link input status.* If not zero, all attempts of filter_frame will fail with the* corresponding code.*/int status_in;/*** Timestamp of the input status change.*/int64_t status_in_pts;/*** Link output status.* If not zero, all attempts of request_frame will fail with the* corresponding code.*/int status_out;#endif /* FF_INTERNAL_FIELDS */};
定义两个filters之间的联接
重点
struct AVFilterLink
{AVFilterContext *src;AVFilterPad *srcpad;AVFilterContext *dst;AVFilterPad *dstpad;struct AVFilterGraph *graph;
};
AVFilterPad-定义filter的输入/输出接⼝
定义filter的输⼊/输出接⼝
重点
struct AVFilterPad
{const char *name;AVFrame *(*get_video_buffer)(AVFilterLink *link, int w, int h);AVFrame *(*get_audio_buffer)(AVFilterLink *link, int nb_samples);int (*filter_frame)(AVFilterLink *link, AVFrame *frame);int (*request_frame)(AVFilterLink *link);
}
完整结构体
/*** A filter pad used for either input or output.*/
struct AVFilterPad {/*** Pad name. The name is unique among inputs and among outputs, but an* input may have the same name as an output. This may be NULL if this* pad has no need to ever be referenced by name.*/const char *name;/*** AVFilterPad type.*/enum AVMediaType type;/*** Callback function to get a video buffer. If NULL, the filter* system will* use ff_default_get_video_buffer().** Input video pads only.*/AVFrame *(*get_video_buffer)(AVFilterLink *link, int w, int h);/*** Callback function to get an audio buffer. If NULL, the filter* system will* use ff_default_get_audio_buffer().** Input audio pads only.*/AVFrame *(*get_audio_buffer)(AVFilterLink *link, int nb_samples);/*** Filtering callback. This is where a filter receives a frame with* audio/video data and should do its processing.** Input pads only.** @return >= 0 on success, a negative AVERROR on error. This function* must ensure that frame is properly unreferenced on error if it* hasn't been passed on to another filter.*/int (*filter_frame)(AVFilterLink *link, AVFrame *frame);/*** Frame poll callback. This returns the number of immediately available* samples. It should return a positive value if the next request_frame()* is guaranteed to return one frame (with no delay).** Defaults to just calling the source poll_frame() method.** Output pads only.*/int (*poll_frame)(AVFilterLink *link);/*** Frame request callback. A call to this should result in some progress* towards producing output over the given link. This should return zero* on success, and another value on error.** Output pads only.*/int (*request_frame)(AVFilterLink *link);/*** Link configuration callback.** For output pads, this should set the link properties such as* width/height. This should NOT set the format property - that is* negotiated between filters by the filter system using the* query_formats() callback before this function is called.** For input pads, this should check the properties of the link, and update* the filter's internal state as necessary.** For both input and output filters, this should return zero on success,* and another value on error.*/int (*config_props)(AVFilterLink *link);/*** The filter expects a fifo to be inserted on its input link,* typically because it has a delay.** input pads only.*/int needs_fifo;/*** The filter expects writable frames from its input link,* duplicating data buffers if needed.** input pads only.*/int needs_writable;
};
AVFilterInOut-过滤器链输⼊/输出的链接列表
/*** A linked-list of the inputs/outputs of the filter chain.** This is mainly useful for avfilter_graph_parse() / avfilter_graph_parse2(),* where it is used to communicate open (unlinked) inputs and outputs from and* to the caller.* This struct specifies, per each not connected pad contained in the graph, the* filter context and the pad index required for establishing a link.*/
typedef struct AVFilterInOut {/** unique name for this input/output in the list */char *name;/** filter context associated to this input/output */AVFilterContext *filter_ctx;/** index of the filt_ctx pad to use for linking */int pad_idx;/** next input/input in the list, NULL if this is the last */struct AVFilterInOut *next;
} AVFilterInOut;
- 在AVFilter模块中定义了AVFilter结构，很个AVFilter都是具有独⽴功能的节点，如scale filter的作⽤就是进⾏图像尺⼨变换，overlay filter的作⽤就是进⾏图像的叠加。
- 这⾥需要重点提的是两个特别的filter，⼀个是buffer，⼀个是buffersink，
- 滤波器buffer代表filter graph中的源头，原始数据就往这个filter节点输⼊的；
- ⽽滤波器buffersink代表filter graph中的输出节点，处理完成的数据从这个filter节点输出
2 函数使⽤
// 获取FFmpeg中定义的filter，调⽤该⽅法前需要先调⽤avfilter_register_all();进⾏滤波器注册
AVFilter avfilter_get_by_name(const char name);
// 往源滤波器buffer中输⼊待处理的数据
int av_buffersrc_add_frame(AVFilterContext ctx, AVFrame frame);
// 从⽬的滤波器buffersink中获取处理完的数据
int av_buffersink_get_frame(AVFilterContext ctx, AVFrame frame);
// 创建⼀个滤波器图filter graph
AVFilterGraph *avfilter_graph_alloc(void);
// 创建⼀个滤波器实例AVFilterContext，并添加到AVFilterGraph中
int avfilter_graph_create_filter(AVFilterContext **filt_ctx, const AVFilter *filt,
const char name, const char args, void *opaque,
AVFilterGraph *graph_ctx);
// 连接两个滤波器节点
int avfilter_link(AVFilterContext *src, unsigned srcpad,
AVFilterContext *dst, unsigned dstpad);
3 AVFilter主体框架流程
在利⽤AVFilter进⾏⾳视频数据处理前先将在进⾏的处理流程绘制出来，现在以FFmpeg filter官⽅⽂档中的⼀个例⼦为例进⾏说明。
[main]
input --> split ---------------------> overlay --> output| ^|[tmp] [flip]|+-----> crop --> vflip -------+
这个例⼦的处理流程如上所示，⾸先使⽤split滤波器将input流分成两路流（main和tmp），然后分别对两路流进⾏处理。对于tmp流，先经过crop滤波器进⾏裁剪处理，再经过flip滤波器进⾏垂直⽅向上的翻转操作，输出的结果命名为flip流。再将main流和flip流输⼊到overlay滤波器进⾏合成操作。上图的input就是上⾯提过的buffer源滤波器，output就是上⾯的提过的buffersink滤波器。上图中每个节点都是⼀个AVFilterContext，每个连线就是AVFliterLink。所有这些信息都统⼀由AVFilterGraph来管理。
实现流程
读入文件
- 准备
yuv数据，放入build目录下
- 打开输入文件，设置分辨率参数
FILE* inFile = NULL;
const char* inFileName = "music.yuv";
fopen_s(&inFile, inFileName, "rb");
if (!inFile) {printf("Fail to open file\n");return -1;
}
int in_width = 1920;
int in_height = 1080;
- 打开输出文件
FILE* outFile = NULL;
const char* outFileName = "out.yuv";
fopen_s(&outFile, outFileName, "wb");
if (!outFile) {printf("Fail to create file for output\n");return -1;
}
配置滤镜
- 首先需要有一个滤镜图
AVFilterGraph，用于管理整个滤镜系统
AVFilterGraph* filter_graph = avfilter_graph_alloc();
if (!filter_graph) {printf("Fail to create filter graph!\n");return -1;
}
- 配置
buffer滤镜，为输入源滤镜 - 需要配置分辨率、像素格式、时间基、像素宽高比（通常是1/1）
char args[512];
sprintf(args,"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",in_width, in_height, AV_PIX_FMT_YUV420P,1, 25, 1, 1);
- 对于每一种滤镜，都是需要先创建滤镜
avfilter_get_by_name - 然后再配置滤镜到滤镜图
avfilter_graph_create_filter
const AVFilter* bufferSrc = avfilter_get_by_name("buffer"); // AVFilterGraph的输入源
AVFilterContext* bufferSrc_ctx;
ret = avfilter_graph_create_filter(&bufferSrc_ctx, bufferSrc, "in", args, NULL, filter_graph);
if (ret < 0) {printf("Fail to create filter bufferSrc\n");return -1;
}
- 配置
buffersink滤镜，为输出滤镜 - 输出滤镜的配置需要使用
AVBufferSinkParams结构体 - 设置像素格式的时候，使用的是一个数组，会选择最合适的一种格式
AVBufferSinkParams *bufferSink_params;
enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE };
bufferSink_params->pixel_fmts = pix_fmts;
- 同样是先创建滤镜，然后再使用配置信息配置滤镜图
const AVFilter* bufferSink = avfilter_get_by_name("buffersink");
ret = avfilter_graph_create_filter(&bufferSink_ctx, bufferSink, "out", NULL,bufferSink_params, filter_graph);
if (ret < 0) {printf("Fail to create filter sink filter\n");return -1;
}
- 配置
split滤镜，这个滤镜用于将一路视频流变为多路视频流输出 - 将视频流分为两路，用于后续的叠加操作
split滤镜中的outputs参数表示将视频流分成多少路输出
const AVFilter *splitFilter = avfilter_get_by_name("split");
AVFilterContext *splitFilter_ctx;
ret = avfilter_graph_create_filter(&splitFilter_ctx, splitFilter, "split", "outputs=2",NULL, filter_graph);
if (ret < 0) {printf("Fail to create split filter\n");return -1;
}
- 配置
crop滤镜，这个滤镜用于裁剪视频 out_w和out_h参数表示输出的视频宽高，x和y表示裁剪的起始点坐标，in_w和in_h表示输入的视频的宽高- 参数之间使用
:隔开
const AVFilter *cropFilter = avfilter_get_by_name("crop");
AVFilterContext *cropFilter_ctx;
ret = avfilter_graph_create_filter(&cropFilter_ctx, cropFilter, "crop","out_w=iw:out_h=ih/2:x=0:y=0", NULL, filter_graph);
if (ret < 0) {printf("Fail to create crop filter\n");return -1;
}
- 配置
vflip滤镜，这个滤镜主要是用于视频的垂直翻转
const AVFilter *vflipFilter = avfilter_get_by_name("vflip");
AVFilterContext *vflipFilter_ctx;
ret = avfilter_graph_create_filter(&vflipFilter_ctx, vflipFilter, "vflip", NULL, NULL, filter_graph);
if (ret < 0) {printf("Fail to create vflip filter\n");return -1;
}
- 配置
overlay滤镜，用于叠加两段视频画面 main_w(W)和main_h(H)表示被叠加视频的宽高、x和y表示叠加的初始位置
const AVFilter *overlayFilter = avfilter_get_by_name("overlay");
AVFilterContext *overlayFilter_ctx;
ret = avfilter_graph_create_filter(&overlayFilter_ctx, overlayFilter, "overlay","x=0:y=H/2", NULL, filter_graph);
if (ret < 0) {printf("Fail to create overlay filter\n");return -1;
}
链接滤镜
- 创建好所有的滤镜之后，需要把滤镜链接起来
- 这里的链接顺序是：
buffer->spilt->video_0->overlay ->buffersink | |video_1->crop->vflip- 根据顺序链接起来
// src filter to split filter
ret = avfilter_link(bufferSrc_ctx, 0, splitFilter_ctx, 0);
if (ret != 0) {printf("Fail to link src filter and split filter\n");return -1;
}
// split filter's first pad to overlay filter's main pad
ret = avfilter_link(splitFilter_ctx, 0, overlayFilter_ctx, 0);
if (ret != 0) {printf("Fail to link split filter and overlay filter main pad\n");return -1;
}
// split filter's second pad to crop filter
ret = avfilter_link(splitFilter_ctx, 1, cropFilter_ctx, 0);
if (ret != 0) {printf("Fail to link split filter's second pad and crop filter\n");return -1;
}
// crop filter to vflip filter
ret = avfilter_link(cropFilter_ctx, 0, vflipFilter_ctx, 0);
if (ret != 0) {printf("Fail to link crop filter and vflip filter\n");return -1;
}
// vflip filter to overlay filter's second pad
ret = avfilter_link(vflipFilter_ctx, 0, overlayFilter_ctx, 1);
if (ret != 0) {printf("Fail to link vflip filter and overlay filter's second pad\n");return -1;
}
// overlay filter to sink filter
ret = avfilter_link(overlayFilter_ctx, 0, bufferSink_ctx, 0);
if (ret != 0) {printf("Fail to link overlay filter and sink filter\n");return -1;
}
生成滤镜图
- 链接之后需要配置滤镜图，生成完整的一套滤镜系统
// check filter graph
ret = avfilter_graph_config(filter_graph, NULL);
if (ret < 0) {printf("Fail in filter graph\n");return -1;
}
- 可以将滤镜图的配置信息
dump到文件中
char *graph_str = avfilter_graph_dump(filter_graph, NULL);
FILE* graphFile = NULL;
fopen_s(&graphFile, "graphFile.txt", "w"); // 打印filtergraph的具体情况
fprintf(graphFile, "%s", graph_str);
av_free(graph_str);
生成输出文件
- 为输入文件和输出文件分配一帧的缓冲区
AVFrame *frame_in = av_frame_alloc();
unsigned char *frame_buffer_in = (unsigned char *)av_malloc(av_image_get_buffer_size(AV_PIX_FMT_YUV420P, in_width, in_height, 1));
av_image_fill_arrays(frame_in->data, frame_in->linesize, frame_buffer_in,AV_PIX_FMT_YUV420P, in_width, in_height, 1);AVFrame *frame_out = av_frame_alloc();
unsigned char *frame_buffer_out = (unsigned char *)av_malloc(av_image_get_buffer_size(AV_PIX_FMT_YUV420P, in_width, in_height, 1));
av_image_fill_arrays(frame_out->data, frame_out->linesize, frame_buffer_out,AV_PIX_FMT_YUV420P, in_width, in_height, 1);
- 设置
frame的参数
frame_in->width = in_width;
frame_in->height = in_height;
frame_in->format = AV_PIX_FMT_YUV420P;
- 读取
yuv输入文件
// 读取yuv数据 1 + 1/4 + 1/4 = 3/2
if (fread(frame_buffer_in, 1, in_width*in_height * 3 / 2, inFile) != in_width*in_height * 3 / 2) {break;
}
//input Y,U,V
frame_in->data[0] = frame_buffer_in;
frame_in->data[1] = frame_buffer_in + in_width*in_height; //偏移 1
frame_in->data[2] = frame_buffer_in + in_width*in_height * 5 / 4; //偏移 1 + 1/4 = 5/4
- 将
frame拷贝到滤镜图中的buffer中，这里无需手动释放frame，内存操作由滤镜系统管理
if (av_buffersrc_add_frame(bufferSrc_ctx, frame_in) < 0) { //把frame拷贝到bufferSrc中，frame无需手动释放printf("Error while add frame.\n");break;
}
- 获取滤镜图处理后
buffersink的数据，拷贝到frame中 - 这里需要手动释放
frame的内存，后续要调用av_frame_unref
/* pull filtered pictures from the filtergraph */
ret = av_buffersink_get_frame(bufferSink_ctx, frame_out); //拷贝BufferSink帧数据到frame
if (ret < 0)break;
- 写入输出的
yuv文件
for (int i = 0; i < frame_out->height; i++) {fwrite(frame_out->data[0] + frame_out->linesize[0] * i, 1, frame_out->width, outFile);
}
for (int i = 0; i < frame_out->height / 2; i++) {fwrite(frame_out->data[1] + frame_out->linesize[1] * i, 1, frame_out->width / 2, outFile);
}
for (int i = 0; i < frame_out->height / 2; i++) {fwrite(frame_out->data[2] + frame_out->linesize[2] * i, 1, frame_out->width / 2, outFile);
}
- 释放输出
frame的内存
av_frame_unref(frame_out); //需要手动释放
结束操作
- 结束的时候需要关闭文件和释放对应
AVFrame结构体内存 - 释放滤镜图内存，会自动释放与之相关的滤镜的内存
fclose(inFile);
fclose(outFile);av_frame_free(&frame_in);
av_frame_free(&frame_out);
avfilter_graph_free(&filter_graph); // 内部去释放AVFilterContext产生的内存
整体代码
#include <stdio.h>#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavfilter/avfilter.h>
#include <libavfilter/buffersink.h>
#include <libavfilter/buffersrc.h>
#include <libavutil/opt.h>
#include <libavutil/imgutils.h>int main(int argc, char** argv)
{int ret = 0;// input yuvFILE* inFile = NULL;const char* inFileName = "music.yuv";fopen_s(&inFile, inFileName, "rb");if (!inFile) {printf("Fail to open file\n");return -1;}int in_width = 1920;int in_height = 1080;// output yuvFILE* outFile = NULL;const char* outFileName = "out.yuv";fopen_s(&outFile, outFileName, "wb");if (!outFile) {printf("Fail to create file for output\n");return -1;}// avfilter_register_all();AVFilterGraph* filter_graph = avfilter_graph_alloc();if (!filter_graph) {printf("Fail to create filter graph!\n");return -1;}// source filterchar args[512];sprintf(args,"video_size=%dx%d:pix_fmt=%d:time_base=%d/%d:pixel_aspect=%d/%d",in_width, in_height, AV_PIX_FMT_YUV420P,1, 25, 1, 1);const AVFilter* bufferSrc = avfilter_get_by_name("buffer"); // AVFilterGraph的输入源AVFilterContext* bufferSrc_ctx;ret = avfilter_graph_create_filter(&bufferSrc_ctx, bufferSrc, "in", args, NULL, filter_graph);if (ret < 0) {printf("Fail to create filter bufferSrc\n");return -1;}// sink filterAVBufferSinkParams *bufferSink_params;AVFilterContext* bufferSink_ctx;const AVFilter* bufferSink = avfilter_get_by_name("buffersink");enum AVPixelFormat pix_fmts[] = { AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE };bufferSink_params = av_buffersink_params_alloc();bufferSink_params->pixel_fmts = pix_fmts;ret = avfilter_graph_create_filter(&bufferSink_ctx, bufferSink, "out", NULL,bufferSink_params, filter_graph);if (ret < 0) {printf("Fail to create filter sink filter\n");return -1;}// split filterconst AVFilter *splitFilter = avfilter_get_by_name("split");AVFilterContext *splitFilter_ctx;ret = avfilter_graph_create_filter(&splitFilter_ctx, splitFilter, "split", "outputs=2",NULL, filter_graph);if (ret < 0) {printf("Fail to create split filter\n");return -1;}// crop filterconst AVFilter *cropFilter = avfilter_get_by_name("crop");AVFilterContext *cropFilter_ctx;ret = avfilter_graph_create_filter(&cropFilter_ctx, cropFilter, "crop","out_w=iw:out_h=ih/2:x=0:y=0", NULL, filter_graph);if (ret < 0) {printf("Fail to create crop filter\n");return -1;}// vflip filterconst AVFilter *vflipFilter = avfilter_get_by_name("vflip");AVFilterContext *vflipFilter_ctx;ret = avfilter_graph_create_filter(&vflipFilter_ctx, vflipFilter, "vflip", NULL, NULL, filter_graph);if (ret < 0) {printf("Fail to create vflip filter\n");return -1;}// overlay filterconst AVFilter *overlayFilter = avfilter_get_by_name("overlay");AVFilterContext *overlayFilter_ctx;ret = avfilter_graph_create_filter(&overlayFilter_ctx, overlayFilter, "overlay","x=0:y=H/2", NULL, filter_graph);if (ret < 0) {printf("Fail to create overlay filter\n");return -1;}// src filter to split filterret = avfilter_link(bufferSrc_ctx, 0, splitFilter_ctx, 0);if (ret != 0) {printf("Fail to link src filter and split filter\n");return -1;}// split filter's first pad to overlay filter's main padret = avfilter_link(splitFilter_ctx, 0, overlayFilter_ctx, 0);if (ret != 0) {printf("Fail to link split filter and overlay filter main pad\n");return -1;}// split filter's second pad to crop filterret = avfilter_link(splitFilter_ctx, 1, cropFilter_ctx, 0);if (ret != 0) {printf("Fail to link split filter's second pad and crop filter\n");return -1;}// crop filter to vflip filterret = avfilter_link(cropFilter_ctx, 0, vflipFilter_ctx, 0);if (ret != 0) {printf("Fail to link crop filter and vflip filter\n");return -1;}// vflip filter to overlay filter's second padret = avfilter_link(vflipFilter_ctx, 0, overlayFilter_ctx, 1);if (ret != 0) {printf("Fail to link vflip filter and overlay filter's second pad\n");return -1;}// overlay filter to sink filterret = avfilter_link(overlayFilter_ctx, 0, bufferSink_ctx, 0);if (ret != 0) {printf("Fail to link overlay filter and sink filter\n");return -1;}// check filter graphret = avfilter_graph_config(filter_graph, NULL);if (ret < 0) {printf("Fail in filter graph\n");return -1;}char *graph_str = avfilter_graph_dump(filter_graph, NULL);FILE* graphFile = NULL;fopen_s(&graphFile, "graphFile.txt", "w"); // 打印filtergraph的具体情况fprintf(graphFile, "%s", graph_str);av_free(graph_str);AVFrame *frame_in = av_frame_alloc();unsigned char *frame_buffer_in = (unsigned char *)av_malloc(av_image_get_buffer_size(AV_PIX_FMT_YUV420P, in_width, in_height, 1));av_image_fill_arrays(frame_in->data, frame_in->linesize, frame_buffer_in,AV_PIX_FMT_YUV420P, in_width, in_height, 1);AVFrame *frame_out = av_frame_alloc();unsigned char *frame_buffer_out = (unsigned char *)av_malloc(av_image_get_buffer_size(AV_PIX_FMT_YUV420P, in_width, in_height, 1));av_image_fill_arrays(frame_out->data, frame_out->linesize, frame_buffer_out,AV_PIX_FMT_YUV420P, in_width, in_height, 1);frame_in->width = in_width;frame_in->height = in_height;frame_in->format = AV_PIX_FMT_YUV420P;uint32_t frame_count = 0;while (1) {// 读取yuv数据 1 + 1/4 + 1/4 = 3/2if (fread(frame_buffer_in, 1, in_width*in_height * 3 / 2, inFile) != in_width*in_height * 3 / 2) {break;}//input Y,U,Vframe_in->data[0] = frame_buffer_in;frame_in->data[1] = frame_buffer_in + in_width*in_height; //偏移 1frame_in->data[2] = frame_buffer_in + in_width*in_height * 5 / 4; //偏移 1 + 1/4 = 5/4if (av_buffersrc_add_frame(bufferSrc_ctx, frame_in) < 0) { //把frame拷贝到bufferSrc中，frame无需手动释放printf("Error while add frame.\n");break;}// filter内部自己处理/* pull filtered pictures from the filtergraph */ret = av_buffersink_get_frame(bufferSink_ctx, frame_out); //拷贝BufferSink帧数据到frameif (ret < 0)break;//output Y,U,Vif (frame_out->format == AV_PIX_FMT_YUV420P) {for (int i = 0; i < frame_out->height; i++) {fwrite(frame_out->data[0] + frame_out->linesize[0] * i, 1, frame_out->width, outFile);}for (int i = 0; i < frame_out->height / 2; i++) {fwrite(frame_out->data[1] + frame_out->linesize[1] * i, 1, frame_out->width / 2, outFile);}for (int i = 0; i < frame_out->height / 2; i++) {fwrite(frame_out->data[2] + frame_out->linesize[2] * i, 1, frame_out->width / 2, outFile);}}++frame_count;if(frame_count % 25 == 0)printf("Process %d frame!\n",frame_count);av_frame_unref(frame_out); //需要手动释放}fclose(inFile);fclose(outFile);av_frame_free(&frame_in);av_frame_free(&frame_out);avfilter_graph_free(&filter_graph); // 内部去释放AVFilterContext产生的内存return 0;
}更多资料：https://github.com/0voice