AOMedia AV1 Codec
temporal_filter.h
1 /*
2  * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3  *
4  * This source code is subject to the terms of the BSD 2 Clause License and
5  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6  * was not distributed with this source code in the LICENSE file, you can
7  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8  * Media Patent License 1.0 was not distributed with this source code in the
9  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10  */
11 
12 #ifndef AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
13 #define AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
14 
15 #include <stdbool.h>
16 
17 #include "aom_util/aom_pthread.h"
18 
19 #ifdef __cplusplus
20 extern "C" {
21 #endif
22 
23 struct AV1_COMP;
24 struct AV1EncoderConfig;
25 struct ThreadData;
26 // TODO(wtc): These two variables are only used in avx2, sse2, neon
27 // implementations, where the block size is still hard coded to TF_BLOCK_SIZE.
28 // This should be fixed to align with the c implementation.
29 #define BH 32
30 #define BW 32
31 
32 // Block size used in temporal filtering.
33 #define TF_BLOCK_SIZE BLOCK_32X32
34 
35 // Window size for temporal filtering.
36 #define TF_WINDOW_LENGTH 5
37 
38 // A constant number, sqrt(pi / 2), used for noise estimation.
39 static const double SQRT_PI_BY_2 = 1.25331413732;
40 
41 // Hyper-parameters used to compute filtering weight. These hyper-parameters can
42 // be tuned for a better performance.
43 // 0. A scale factor used in temporal filtering to raise the filter weight from
44 // `double` with range [0, 1] to `int` with range [0, 1000].
45 #define TF_WEIGHT_SCALE 1000
46 // 1. Weight factor used to balance the weighted-average between window error
47 // and block error. The weight is for window error while the weight for block
48 // error is always set as 1.
49 #define TF_WINDOW_BLOCK_BALANCE_WEIGHT 5
50 // 2. Threshold for using q to adjust the filtering weight. Concretely, when
51 // using a small q (high bitrate), we would like to reduce the filtering
52 // strength such that more detailed information can be preserved. Hence, when
53 // q is smaller than this threshold, we will adjust the filtering weight
54 // based on the q-value.
55 #define TF_Q_DECAY_THRESHOLD 20
56 // 3. Normalization factor used to normalize the motion search error. Since the
57 // motion search error can be large and uncontrollable, we will simply
58 // normalize it before using it to compute the filtering weight.
59 #define TF_SEARCH_ERROR_NORM_WEIGHT 20
60 // 4. Threshold for using `arnr_strength` to adjust the filtering strength.
61 // Concretely, users can use `arnr_strength` arguments to control the
62 // strength of temporal filtering. When `arnr_strength` is small enough (
63 // i.e., smaller than this threshold), we will adjust the filtering weight
64 // based on the strength value.
65 #define TF_STRENGTH_THRESHOLD 4
66 // 5. Threshold for using motion search distance to adjust the filtering weight.
67 // Concretely, larger motion search vector leads to a higher probability of
68 // unreliable search. Hence, we would like to reduce the filtering strength
69 // when the distance is large enough. Considering that the distance actually
70 // relies on the frame size, this threshold is also a resolution-based
71 // threshold. Taking 720p videos as an instance, if this field equals to 0.1,
72 // then the actual threshold will be 720 * 0.1 = 72. Similarly, the threshold
73 // for 360p videos will be 360 * 0.1 = 36.
74 #define TF_SEARCH_DISTANCE_THRESHOLD 0.1
75 // 6. Threshold to identify if the q is in a relative high range.
76 // Above this cutoff q, a stronger filtering is applied.
77 // For a high q, the quantization throws away more information, and thus a
78 // stronger filtering is less likely to distort the encoded quality, while a
79 // stronger filtering could reduce bit rates.
80 // Ror a low q, more details are expected to be retained. Filtering is thus
81 // more conservative.
82 #define TF_QINDEX_CUTOFF 128
83 
84 #define NOISE_ESTIMATION_EDGE_THRESHOLD 50
85 
86 // Sum and SSE source vs filtered frame difference returned by
87 // temporal filter.
88 typedef struct {
89  int64_t sum;
90  int64_t sse;
91 } FRAME_DIFF;
92 
98 typedef struct {
102  YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS];
107 
112 
124  struct scale_factors sf;
128  double noise_levels[MAX_MB_PLANE];
132  int num_pels;
136  int mb_rows;
140  int mb_cols;
148  int q_factor;
150 
156 #define TF_INFO_BUF_COUNT 2
157 
161 typedef struct TEMPORAL_FILTER_INFO {
172  YV12_BUFFER_CONFIG tf_buf[TF_INFO_BUF_COUNT];
173 
184  FRAME_DIFF frame_diff[TF_INFO_BUF_COUNT];
188  int tf_buf_gf_index[TF_INFO_BUF_COUNT];
192  int tf_buf_display_index_offset[TF_INFO_BUF_COUNT];
196  int tf_buf_valid[TF_INFO_BUF_COUNT];
198 
204 int av1_is_temporal_filter_on(const struct AV1EncoderConfig *oxcf);
205 
212 bool av1_tf_info_alloc(TEMPORAL_FILTER_INFO *tf_info,
213  const struct AV1_COMP *cpi);
214 
218 void av1_tf_info_free(TEMPORAL_FILTER_INFO *tf_info);
219 
223 void av1_tf_info_reset(TEMPORAL_FILTER_INFO *tf_info);
224 
230 void av1_tf_info_filtering(TEMPORAL_FILTER_INFO *tf_info, struct AV1_COMP *cpi,
231  const GF_GROUP *gf_group);
232 
239 YV12_BUFFER_CONFIG *av1_tf_info_get_filtered_buf(TEMPORAL_FILTER_INFO *tf_info,
240  int gf_index,
241  FRAME_DIFF *frame_diff);
242 
245 // Data related to temporal filtering.
246 typedef struct {
247  // Source vs filtered frame error.
248  FRAME_DIFF diff;
249  // Pointer to temporary block info used to store state in temporal filtering
250  // process.
251  MB_MODE_INFO *tmp_mbmi;
252  // Pointer to accumulator buffer used in temporal filtering process.
253  uint32_t *accum;
254  // Pointer to count buffer used in temporal filtering process.
255  uint16_t *count;
256  // Pointer to predictor used in temporal filtering process.
257  uint8_t *pred;
258 } TemporalFilterData;
259 
260 // Data related to temporal filter multi-thread synchronization.
261 typedef struct {
262 #if CONFIG_MULTITHREAD
263  // Mutex lock used for dispatching jobs.
264  pthread_mutex_t *mutex_;
265 #endif // CONFIG_MULTITHREAD
266  // Next temporal filter block row to be filtered.
267  int next_tf_row;
268  // Initialized to false, set to true by the worker thread that encounters an
269  // error in order to abort the processing of other worker threads.
270  bool tf_mt_exit;
271 } AV1TemporalFilterSync;
272 
273 // Estimates noise level from a given frame using a single plane (Y, U, or V).
274 // This is an adaptation of the mehtod in the following paper:
275 // Shen-Chuan Tai, Shih-Ming Yang, "A fast method for image noise
276 // estimation using Laplacian operator and adaptive edge detection",
277 // Proc. 3rd International Symposium on Communications, Control and
278 // Signal Processing, 2008, St Julians, Malta.
279 // Inputs:
280 // frame: Pointer to the frame to estimate noise level from.
281 // noise_level: Pointer to store the estimated noise.
282 // plane_from: Index of the starting plane used for noise estimation.
283 // Commonly, 0 for Y-plane, 1 for U-plane, and 2 for V-plane.
284 // plane_to: Index of the end plane used for noise estimation.
285 // bit_depth: Actual bit-depth instead of the encoding bit-depth of the frame.
286 // edge_thresh: Edge threshold.
287 void av1_estimate_noise_level(const YV12_BUFFER_CONFIG *frame,
288  double *noise_level, int plane_from, int plane_to,
289  int bit_depth, int edge_thresh);
303 void av1_tf_do_filtering_row(struct AV1_COMP *cpi, struct ThreadData *td,
304  int mb_row);
305 
330 void av1_temporal_filter(struct AV1_COMP *cpi,
331  const int filter_frame_lookahead_idx,
332  int gf_frame_index, FRAME_DIFF *frame_diff,
333  YV12_BUFFER_CONFIG *output_frame);
334 
350  const FRAME_DIFF *frame_diff, int q_index,
351  aom_bit_depth_t bit_depth);
352 
354 // Helper function to get `q` used for encoding.
355 int av1_get_q(const struct AV1_COMP *cpi);
356 
357 // Allocates memory for members of TemporalFilterData.
358 // Inputs:
359 // tf_data: Pointer to the structure containing temporal filter related data.
360 // num_pels: Number of pixels in the block across all planes.
361 // is_high_bitdepth: Whether the frame is high-bitdepth or not.
362 // Returns:
363 // True if allocation is successful and false otherwise.
364 static AOM_INLINE bool tf_alloc_and_reset_data(TemporalFilterData *tf_data,
365  int num_pels,
366  int is_high_bitdepth) {
367  tf_data->tmp_mbmi = (MB_MODE_INFO *)aom_calloc(1, sizeof(*tf_data->tmp_mbmi));
368  tf_data->accum =
369  (uint32_t *)aom_memalign(16, num_pels * sizeof(*tf_data->accum));
370  tf_data->count =
371  (uint16_t *)aom_memalign(16, num_pels * sizeof(*tf_data->count));
372  if (is_high_bitdepth)
373  tf_data->pred = CONVERT_TO_BYTEPTR(
374  aom_memalign(32, num_pels * 2 * sizeof(*tf_data->pred)));
375  else
376  tf_data->pred =
377  (uint8_t *)aom_memalign(32, num_pels * sizeof(*tf_data->pred));
378  // In case of an allocation failure, other successfully allocated buffers will
379  // be freed by the tf_dealloc_data() call in encoder_destroy().
380  if (!(tf_data->tmp_mbmi && tf_data->accum && tf_data->count && tf_data->pred))
381  return false;
382  memset(&tf_data->diff, 0, sizeof(tf_data->diff));
383  return true;
384 }
385 
386 // Setup macroblockd params for temporal filtering process.
387 // Inputs:
388 // mbd: Pointer to the block for filtering.
389 // tf_data: Pointer to the structure containing temporal filter related data.
390 // scale: Scaling factor.
391 // Returns:
392 // Nothing will be returned. Contents of mbd will be modified.
393 static AOM_INLINE void tf_setup_macroblockd(MACROBLOCKD *mbd,
394  TemporalFilterData *tf_data,
395  const struct scale_factors *scale) {
396  mbd->block_ref_scale_factors[0] = scale;
397  mbd->block_ref_scale_factors[1] = scale;
398  mbd->mi = &tf_data->tmp_mbmi;
399  mbd->mi[0]->motion_mode = SIMPLE_TRANSLATION;
400 }
401 
402 // Deallocates the memory allocated for members of TemporalFilterData.
403 // Inputs:
404 // tf_data: Pointer to the structure containing temporal filter related data.
405 // is_high_bitdepth: Whether the frame is high-bitdepth or not.
406 // Returns:
407 // Nothing will be returned.
408 static AOM_INLINE void tf_dealloc_data(TemporalFilterData *tf_data,
409  int is_high_bitdepth) {
410  if (is_high_bitdepth)
411  tf_data->pred = (uint8_t *)CONVERT_TO_SHORTPTR(tf_data->pred);
412  aom_free(tf_data->tmp_mbmi);
413  tf_data->tmp_mbmi = NULL;
414  aom_free(tf_data->accum);
415  tf_data->accum = NULL;
416  aom_free(tf_data->count);
417  tf_data->count = NULL;
418  aom_free(tf_data->pred);
419  tf_data->pred = NULL;
420 }
421 
422 // Saves the state prior to temporal filter process.
423 // Inputs:
424 // mbd: Pointer to the block for filtering.
425 // input_mbmi: Backup block info to save input state.
426 // input_buffer: Backup buffer pointer to save input state.
427 // num_planes: Number of planes.
428 // Returns:
429 // Nothing will be returned. Contents of input_mbmi and input_buffer will be
430 // modified.
431 static INLINE void tf_save_state(MACROBLOCKD *mbd, MB_MODE_INFO ***input_mbmi,
432  uint8_t **input_buffer, int num_planes) {
433  for (int i = 0; i < num_planes; i++) {
434  input_buffer[i] = mbd->plane[i].pre[0].buf;
435  }
436  *input_mbmi = mbd->mi;
437 }
438 
439 // Restores the initial state after temporal filter process.
440 // Inputs:
441 // mbd: Pointer to the block for filtering.
442 // input_mbmi: Backup block info from where input state is restored.
443 // input_buffer: Backup buffer pointer from where input state is restored.
444 // num_planes: Number of planes.
445 // Returns:
446 // Nothing will be returned. Contents of mbd will be modified.
447 static INLINE void tf_restore_state(MACROBLOCKD *mbd, MB_MODE_INFO **input_mbmi,
448  uint8_t **input_buffer, int num_planes) {
449  for (int i = 0; i < num_planes; i++) {
450  mbd->plane[i].pre[0].buf = input_buffer[i];
451  }
452  mbd->mi = input_mbmi;
453 }
454 
456 #ifdef __cplusplus
457 } // extern "C"
458 #endif
459 
460 #endif // AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
void av1_temporal_filter(struct AV1_COMP *cpi, const int filter_frame_lookahead_idx, int gf_frame_index, FRAME_DIFF *frame_diff, YV12_BUFFER_CONFIG *output_frame)
Performs temporal filtering if needed on a source frame. For example to create a filtered alternate r...
YV12_BUFFER_CONFIG tf_buf[2]
Definition: temporal_filter.h:172
int mb_cols
Definition: temporal_filter.h:140
Parameters related to temporal filtering.
Definition: temporal_filter.h:98
int num_pels
Definition: temporal_filter.h:132
struct macroblockd_plane plane[3]
Definition: blockd.h:606
int compute_frame_diff
Definition: temporal_filter.h:120
Data related to the current GF/ARF group and the individual frames within the group.
Definition: firstpass.h:354
int filter_frame_idx
Definition: temporal_filter.h:116
const struct scale_factors * block_ref_scale_factors[2]
Definition: blockd.h:687
MOTION_MODE motion_mode
The motion mode used by the inter prediction.
Definition: blockd.h:250
int tf_buf_gf_index[2]
Definition: temporal_filter.h:188
int tf_buf_display_index_offset[2]
Definition: temporal_filter.h:192
int mb_rows
Definition: temporal_filter.h:136
YV12 frame buffer data structure.
Definition: yv12config.h:46
int tf_buf_valid[2]
Definition: temporal_filter.h:196
MB_MODE_INFO ** mi
Definition: blockd.h:617
Variables related to current coding block.
Definition: blockd.h:570
int is_temporal_filter_on
Definition: temporal_filter.h:167
int q_factor
Definition: temporal_filter.h:148
Top level encoder structure.
Definition: encoder.h:2866
enum aom_bit_depth aom_bit_depth_t
Bit depth for codecThis enumeration determines the bit depth of the codec.
FRAME_DIFF frame_diff[2]
Definition: temporal_filter.h:184
YV12_BUFFER_CONFIG tf_buf_second_arf
Definition: temporal_filter.h:180
Temporal filter info for a gop.
Definition: temporal_filter.h:161
Stores the prediction/txfm mode of the current coding block.
Definition: blockd.h:222
int num_frames
Definition: temporal_filter.h:106
int is_highbitdepth
Definition: temporal_filter.h:144
void av1_tf_do_filtering_row(struct AV1_COMP *cpi, struct ThreadData *td, int mb_row)
Does temporal filter for a given macroblock row.
Definition: temporal_filter.c:859
int av1_check_show_filtered_frame(const YV12_BUFFER_CONFIG *frame, const FRAME_DIFF *frame_diff, int q_index, aom_bit_depth_t bit_depth)
Check whether a filtered frame can be show directly.
Main encoder configuration data structure.
Definition: encoder.h:916
YV12_BUFFER_CONFIG * output_frame
Definition: temporal_filter.h:111