FFmpeg  4.3
vf_derain.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019 Xuewei Meng
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * Filter implementing image derain filter using deep convolutional networks.
24  * http://openaccess.thecvf.com/content_ECCV_2018/html/Xia_Li_Recurrent_Squeeze-and-Excitation_Context_ECCV_2018_paper.html
25  */
26 
27 #include "libavformat/avio.h"
28 #include "libavutil/opt.h"
29 #include "avfilter.h"
30 #include "dnn_interface.h"
31 #include "formats.h"
32 #include "internal.h"
33 
34 typedef struct DRContext {
35  const AVClass *class;
36 
44 } DRContext;
45 
46 #define CLIP(x, min, max) (x < min ? min : (x > max ? max : x))
47 #define OFFSET(x) offsetof(DRContext, x)
48 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
49 static const AVOption derain_options[] = {
50  { "filter_type", "filter type(derain/dehaze)", OFFSET(filter_type), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS, "type" },
51  { "derain", "derain filter flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "type" },
52  { "dehaze", "dehaze filter flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "type" },
53  { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS, "backend" },
54  { "native", "native backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "backend" },
55 #if (CONFIG_LIBTENSORFLOW == 1)
56  { "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" },
57 #endif
58  { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
59  { NULL }
60 };
61 
62 AVFILTER_DEFINE_CLASS(derain);
63 
65 {
67  const enum AVPixelFormat pixel_fmts[] = {
70  };
71 
72  formats = ff_make_format_list(pixel_fmts);
73 
74  return ff_set_common_formats(ctx, formats);
75 }
76 
77 static int config_inputs(AVFilterLink *inlink)
78 {
79  AVFilterContext *ctx = inlink->dst;
80  DRContext *dr_context = ctx->priv;
81  const char *model_output_name = "y";
82  DNNReturnType result;
83 
84  dr_context->input.width = inlink->w;
85  dr_context->input.height = inlink->h;
86  dr_context->input.channels = 3;
87 
88  result = (dr_context->model->set_input_output)(dr_context->model->model, &dr_context->input, "x", &model_output_name, 1);
89  if (result != DNN_SUCCESS) {
90  av_log(ctx, AV_LOG_ERROR, "could not set input and output for the model\n");
91  return AVERROR(EIO);
92  }
93 
94  return 0;
95 }
96 
97 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
98 {
99  AVFilterContext *ctx = inlink->dst;
100  AVFilterLink *outlink = ctx->outputs[0];
101  DRContext *dr_context = ctx->priv;
102  DNNReturnType dnn_result;
103 
104  AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
105  if (!out) {
106  av_log(ctx, AV_LOG_ERROR, "could not allocate memory for output frame\n");
107  av_frame_free(&in);
108  return AVERROR(ENOMEM);
109  }
110 
111  av_frame_copy_props(out, in);
112 
113  for (int i = 0; i < in->height; i++){
114  for(int j = 0; j < in->width * 3; j++){
115  int k = i * in->linesize[0] + j;
116  int t = i * in->width * 3 + j;
117  ((float *)dr_context->input.data)[t] = in->data[0][k] / 255.0;
118  }
119  }
120 
121  dnn_result = (dr_context->dnn_module->execute_model)(dr_context->model, &dr_context->output, 1);
122  if (dnn_result != DNN_SUCCESS){
123  av_log(ctx, AV_LOG_ERROR, "failed to execute model\n");
124  return AVERROR(EIO);
125  }
126 
127  out->height = dr_context->output.height;
128  out->width = dr_context->output.width;
129  outlink->h = dr_context->output.height;
130  outlink->w = dr_context->output.width;
131 
132  for (int i = 0; i < out->height; i++){
133  for(int j = 0; j < out->width * 3; j++){
134  int k = i * out->linesize[0] + j;
135  int t = i * out->width * 3 + j;
136  out->data[0][k] = CLIP((int)((((float *)dr_context->output.data)[t]) * 255), 0, 255);
137  }
138  }
139 
140  av_frame_free(&in);
141 
142  return ff_filter_frame(outlink, out);
143 }
144 
146 {
147  DRContext *dr_context = ctx->priv;
148 
149  dr_context->input.dt = DNN_FLOAT;
150  dr_context->dnn_module = ff_get_dnn_module(dr_context->backend_type);
151  if (!dr_context->dnn_module) {
152  av_log(ctx, AV_LOG_ERROR, "could not create DNN module for requested backend\n");
153  return AVERROR(ENOMEM);
154  }
155  if (!dr_context->model_filename) {
156  av_log(ctx, AV_LOG_ERROR, "model file for network is not specified\n");
157  return AVERROR(EINVAL);
158  }
159  if (!dr_context->dnn_module->load_model) {
160  av_log(ctx, AV_LOG_ERROR, "load_model for network is not specified\n");
161  return AVERROR(EINVAL);
162  }
163 
164  dr_context->model = (dr_context->dnn_module->load_model)(dr_context->model_filename);
165  if (!dr_context->model) {
166  av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n");
167  return AVERROR(EINVAL);
168  }
169 
170  return 0;
171 }
172 
174 {
175  DRContext *dr_context = ctx->priv;
176 
177  if (dr_context->dnn_module) {
178  (dr_context->dnn_module->free_model)(&dr_context->model);
179  av_freep(&dr_context->dnn_module);
180  }
181 }
182 
183 static const AVFilterPad derain_inputs[] = {
184  {
185  .name = "default",
186  .type = AVMEDIA_TYPE_VIDEO,
187  .config_props = config_inputs,
188  .filter_frame = filter_frame,
189  },
190  { NULL }
191 };
192 
193 static const AVFilterPad derain_outputs[] = {
194  {
195  .name = "default",
196  .type = AVMEDIA_TYPE_VIDEO,
197  },
198  { NULL }
199 };
200 
202  .name = "derain",
203  .description = NULL_IF_CONFIG_SMALL("Apply derain filter to the input."),
204  .priv_size = sizeof(DRContext),
205  .init = init,
206  .uninit = uninit,
208  .inputs = derain_inputs,
209  .outputs = derain_outputs,
210  .priv_class = &derain_class,
212 };
void * model
Definition: dnn_interface.h:45
#define NULL
Definition: coverity.c:32
Buffered I/O operations.
This structure describes decoded (raw) audio or video data.
Definition: frame.h:300
AVOption.
Definition: opt.h:246
void(* free_model)(DNNModel **model)
Definition: dnn_interface.h:61
Main libavfilter public API header.
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:68
static const AVOption derain_options[]
Definition: vf_derain.c:49
int channels
Definition: dnn_interface.h:40
static int config_inputs(AVFilterLink *inlink)
Definition: vf_derain.c:77
DNNBackendType backend_type
Definition: vf_derain.c:39
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:99
AVFilterFormats * ff_make_format_list(const int *fmts)
Create a list of supported formats.
Definition: formats.c:283
#define AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC
Some filters support a generic "enable" expression option that can be used to enable or disable a fil...
Definition: avfilter.h:125
const char * name
Pad name.
Definition: internal.h:60
DNNModel *(* load_model)(const char *model_filename)
Definition: dnn_interface.h:57
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1075
#define av_cold
Definition: attributes.h:88
char * model_filename
Definition: vf_derain.c:38
AVOptions.
DNNData input
Definition: vf_derain.c:42
AVFILTER_DEFINE_CLASS(derain)
DNNBackendType
Definition: dnn_interface.h:33
#define av_log(a,...)
A filter pad used for either input or output.
Definition: internal.h:54
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:269
int width
Definition: frame.h:358
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:600
int height
Definition: dnn_interface.h:40
#define AVERROR(e)
Definition: error.h:43
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:203
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:186
void * data
Definition: dnn_interface.h:38
void * priv
private data for use by the filter
Definition: avfilter.h:353
static av_cold int init(AVFilterContext *ctx)
Definition: vf_derain.c:145
static av_cold void uninit(AVFilterContext *ctx)
Definition: vf_derain.c:173
DNNData output
Definition: vf_derain.c:43
AVFilter ff_vf_derain
Definition: vf_derain.c:201
DNNModel * model
Definition: vf_derain.c:41
AVFormatContext * ctx
Definition: movenc.c:48
int filter_type
Definition: vf_derain.c:37
DNN inference engine interface.
static const AVFilterPad inputs[]
Definition: af_acontrast.c:193
DNNReturnType
Definition: dnn_interface.h:31
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
#define OFFSET(x)
Definition: vf_derain.c:47
#define CLIP(x, min, max)
Definition: vf_derain.c:46
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:331
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi - 0x80) *(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi - 0x80) *(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t, *(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t, *(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t, *(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t, *(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31)))) #define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac) { } void ff_audio_convert_free(AudioConvert **ac) { if(! *ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);} AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map) { AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method !=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2) { ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc) { av_free(ac);return NULL;} return ac;} in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar) { ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar ? ac->channels :1;} else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;} int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in) { int use_generic=1;int len=in->nb_samples;int p;if(ac->dc) { av_log(ac->avr, AV_LOG_TRACE, "%d samples - audio_convert: %s to %s (dithered)\", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
Describe the class of an AVClass context structure.
Definition: log.h:67
Filter definition.
Definition: avfilter.h:144
const char * name
Filter name.
Definition: avfilter.h:148
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:350
DNNModule * ff_get_dnn_module(DNNBackendType backend_type)
Definition: dnn_interface.c:31
#define flags(name, subs,...)
Definition: cbs_av1.c:564
DNNReturnType(* execute_model)(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
Definition: dnn_interface.h:59
static const AVFilterPad derain_inputs[]
Definition: vf_derain.c:183
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:314
static int filter_frame(AVFilterLink *inlink, AVFrame *in)
Definition: vf_derain.c:97
#define FLAGS
Definition: vf_derain.c:48
static int query_formats(AVFilterContext *ctx)
Definition: vf_derain.c:64
A list of supported formats for one end of a filter link.
Definition: formats.h:64
An instance of a filter.
Definition: avfilter.h:338
DNNModule * dnn_module
Definition: vf_derain.c:40
int height
Definition: frame.h:358
FILE * out
Definition: movenc.c:54
#define av_freep(p)
DNNReturnType(* set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
Definition: dnn_interface.h:51
formats
Definition: signature.h:48
internal API functions
DNNDataType dt
Definition: dnn_interface.h:39
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
static const AVFilterPad derain_outputs[]
Definition: vf_derain.c:193
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:659