AOMedia Codec SDK
aom_cx_set_ref
1 /*
2  * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3  *
4  * This source code is subject to the terms of the BSD 2 Clause License and
5  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6  * was not distributed with this source code in the LICENSE file, you can
7  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8  * Media Patent License 1.0 was not distributed with this source code in the
9  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10  */
11 
12 // AV1 Set Reference Frame
13 // ============================
14 //
15 // This is an example demonstrating how to overwrite the AV1 encoder's
16 // internal reference frame. In the sample we set the last frame to the
17 // current frame. This technique could be used to bounce between two cameras.
18 //
19 // The decoder would also have to set the reference frame to the same value
20 // on the same frame, or the video will become corrupt. The 'test_decode'
21 // variable is set to 1 in this example that tests if the encoder and decoder
22 // results are matching.
23 //
24 // Usage
25 // -----
26 // This example encodes a raw video. And the last argument passed in specifies
27 // the frame number to update the reference frame on. For example, run
28 // examples/aom_cx_set_ref av1 352 288 in.yuv out.ivf 4 30
29 // The parameter is parsed as follows:
30 //
31 //
32 // Extra Variables
33 // ---------------
34 // This example maintains the frame number passed on the command line
35 // in the `update_frame_num` variable.
36 //
37 //
38 // Configuration
39 // -------------
40 //
41 // The reference frame is updated on the frame specified on the command
42 // line.
43 //
44 // Observing The Effects
45 // ---------------------
46 // The encoder and decoder results should be matching when the same reference
47 // frame setting operation is done in both encoder and decoder. Otherwise,
48 // the encoder/decoder mismatch would be seen.
49 
50 #include <stdio.h>
51 #include <stdlib.h>
52 #include <string.h>
53 
54 #include "aom/aom_decoder.h"
55 #include "aom/aom_encoder.h"
56 #include "aom/aomcx.h"
57 #include "common/tools_common.h"
58 #include "common/video_writer.h"
59 #include "examples/encoder_util.h"
60 
61 #define AOM_BORDER_IN_PIXELS 288
62 
63 static const char *exec_name;
64 
65 void usage_exit() {
66  fprintf(stderr,
67  "Usage: %s <codec> <width> <height> <infile> <outfile> "
68  "<frame> <limit(optional)>\n",
69  exec_name);
70  exit(EXIT_FAILURE);
71 }
72 
73 static void testing_decode(aom_codec_ctx_t *encoder, aom_codec_ctx_t *decoder,
74  unsigned int frame_out, int *mismatch_seen) {
75  aom_image_t enc_img, dec_img;
76 
77  if (*mismatch_seen) return;
78 
79  /* Get the internal reference frame */
80  if (aom_codec_control(encoder, AV1_GET_NEW_FRAME_IMAGE, &enc_img))
81  die_codec(encoder, "Failed to get encoder reference frame");
82  if (aom_codec_control(decoder, AV1_GET_NEW_FRAME_IMAGE, &dec_img))
83  die_codec(decoder, "Failed to get decoder reference frame");
84 
85  if ((enc_img.fmt & AOM_IMG_FMT_HIGHBITDEPTH) !=
86  (dec_img.fmt & AOM_IMG_FMT_HIGHBITDEPTH)) {
87  if (enc_img.fmt & AOM_IMG_FMT_HIGHBITDEPTH) {
88  aom_image_t enc_hbd_img;
89  aom_img_alloc(&enc_hbd_img, enc_img.fmt - AOM_IMG_FMT_HIGHBITDEPTH,
90  enc_img.d_w, enc_img.d_h, 16);
91  aom_img_truncate_16_to_8(&enc_hbd_img, &enc_img);
92  enc_img = enc_hbd_img;
93  }
94  if (dec_img.fmt & AOM_IMG_FMT_HIGHBITDEPTH) {
95  aom_image_t dec_hbd_img;
96  aom_img_alloc(&dec_hbd_img, dec_img.fmt - AOM_IMG_FMT_HIGHBITDEPTH,
97  dec_img.d_w, dec_img.d_h, 16);
98  aom_img_truncate_16_to_8(&dec_hbd_img, &dec_img);
99  dec_img = dec_hbd_img;
100  }
101  }
102 
103  if (!aom_compare_img(&enc_img, &dec_img)) {
104  int y[4], u[4], v[4];
105  if (enc_img.fmt & AOM_IMG_FMT_HIGHBITDEPTH) {
106  aom_find_mismatch_high(&enc_img, &dec_img, y, u, v);
107  } else {
108  aom_find_mismatch(&enc_img, &dec_img, y, u, v);
109  }
110 
111  printf(
112  "Encode/decode mismatch on frame %d at"
113  " Y[%d, %d] {%d/%d},"
114  " U[%d, %d] {%d/%d},"
115  " V[%d, %d] {%d/%d}",
116  frame_out, y[0], y[1], y[2], y[3], u[0], u[1], u[2], u[3], v[0], v[1],
117  v[2], v[3]);
118  *mismatch_seen = 1;
119  }
120 
121  aom_img_free(&enc_img);
122  aom_img_free(&dec_img);
123 }
124 
125 static int encode_frame(aom_codec_ctx_t *ecodec, aom_image_t *img,
126  unsigned int frame_in, AvxVideoWriter *writer,
127  int test_decode, aom_codec_ctx_t *dcodec,
128  unsigned int *frame_out, int *mismatch_seen,
129  aom_image_t *ext_ref) {
130  int got_pkts = 0;
131  aom_codec_iter_t iter = NULL;
132  const aom_codec_cx_pkt_t *pkt = NULL;
133  int got_data;
134  const aom_codec_err_t res = aom_codec_encode(ecodec, img, frame_in, 1, 0);
135  if (res != AOM_CODEC_OK) die_codec(ecodec, "Failed to encode frame");
136 
137  got_data = 0;
138 
139  while ((pkt = aom_codec_get_cx_data(ecodec, &iter)) != NULL) {
140  got_pkts = 1;
141 
142  if (pkt->kind == AOM_CODEC_CX_FRAME_PKT) {
143  const int keyframe = (pkt->data.frame.flags & AOM_FRAME_IS_KEY) != 0;
144 
145  if (!(pkt->data.frame.flags & AOM_FRAME_IS_FRAGMENT)) {
146  *frame_out += 1;
147  }
148 
149  if (!aom_video_writer_write_frame(writer, pkt->data.frame.buf,
150  pkt->data.frame.sz,
151  pkt->data.frame.pts)) {
152  die_codec(ecodec, "Failed to write compressed frame");
153  }
154  printf(keyframe ? "K" : ".");
155  fflush(stdout);
156  got_data = 1;
157 
158  // Decode 1 frame.
159  if (test_decode) {
160  if (aom_codec_decode(dcodec, pkt->data.frame.buf,
161  (unsigned int)pkt->data.frame.sz, NULL))
162  die_codec(dcodec, "Failed to decode frame.");
163 
164  // Copy out first decoded frame, and use it as reference later.
165  if (*frame_out == 1 && ext_ref != NULL)
166  if (aom_codec_control(dcodec, AV1_GET_NEW_FRAME_IMAGE, ext_ref))
167  die_codec(dcodec, "Failed to get decoder new frame");
168  }
169  }
170  }
171 
172  // Mismatch checking
173  if (got_data && test_decode) {
174  testing_decode(ecodec, dcodec, *frame_out, mismatch_seen);
175  }
176 
177  return got_pkts;
178 }
179 
180 int main(int argc, char **argv) {
181  FILE *infile = NULL;
182  // Encoder
183  aom_codec_ctx_t ecodec;
185  unsigned int frame_in = 0;
186  aom_image_t raw;
187  aom_image_t raw_shift;
188  aom_image_t ext_ref;
189  aom_codec_err_t res;
190  AvxVideoInfo info;
191  AvxVideoWriter *writer = NULL;
192  const AvxInterface *encoder = NULL;
193  int flags = 0;
194  int allocated_raw_shift = 0;
197 
198  // Test encoder/decoder mismatch.
199  int test_decode = 1;
200  // Decoder
201  aom_codec_ctx_t dcodec;
202  unsigned int frame_out = 0;
203 
204  // The frame number to set reference frame on
205  unsigned int update_frame_num = 0;
206  int mismatch_seen = 0;
207 
208  const int fps = 30;
209  const int bitrate = 500;
210 
211  const char *codec_arg = NULL;
212  const char *width_arg = NULL;
213  const char *height_arg = NULL;
214  const char *infile_arg = NULL;
215  const char *outfile_arg = NULL;
216  const char *update_frame_num_arg = NULL;
217  unsigned int limit = 0;
218  exec_name = argv[0];
219 
220  // Clear explicitly, as simply assigning "{ 0 }" generates
221  // "missing-field-initializers" warning in some compilers.
222  memset(&ecodec, 0, sizeof(ecodec));
223  memset(&cfg, 0, sizeof(cfg));
224  memset(&info, 0, sizeof(info));
225 
226  if (argc < 7) die("Invalid number of arguments");
227 
228  codec_arg = argv[1];
229  width_arg = argv[2];
230  height_arg = argv[3];
231  infile_arg = argv[4];
232  outfile_arg = argv[5];
233  update_frame_num_arg = argv[6];
234 
235  encoder = get_aom_encoder_by_name(codec_arg);
236  if (!encoder) die("Unsupported codec.");
237 
238  update_frame_num = (unsigned int)strtoul(update_frame_num_arg, NULL, 0);
239  // In AV1, the reference buffers (cm->buffer_pool->frame_bufs[i].buf) are
240  // allocated while calling aom_codec_encode(), thus, setting reference for
241  // 1st frame isn't supported.
242  if (update_frame_num <= 1) {
243  die("Couldn't parse frame number '%s'\n", update_frame_num_arg);
244  }
245 
246  if (argc > 7) {
247  limit = (unsigned int)strtoul(argv[7], NULL, 0);
248  if (update_frame_num > limit)
249  die("Update frame number couldn't larger than limit\n");
250  }
251 
252  info.codec_fourcc = encoder->fourcc;
253  info.frame_width = (int)strtol(width_arg, NULL, 0);
254  info.frame_height = (int)strtol(height_arg, NULL, 0);
255  info.time_base.numerator = 1;
256  info.time_base.denominator = fps;
257 
258  if (info.frame_width <= 0 || info.frame_height <= 0) {
259  die("Invalid frame size: %dx%d", info.frame_width, info.frame_height);
260  }
261 
262  // In this test, the bit depth of input video is 8-bit, and the input format
263  // is AOM_IMG_FMT_I420.
264  if (!aom_img_alloc(&raw, raw_fmt, info.frame_width, info.frame_height, 32)) {
265  die("Failed to allocate image.");
266  }
267 
268  if (!CONFIG_LOWBITDEPTH) ref_fmt |= AOM_IMG_FMT_HIGHBITDEPTH;
269  // Allocate memory with the border so that it can be used as a reference.
270  if (!aom_img_alloc_with_border(&ext_ref, ref_fmt, info.frame_width,
271  info.frame_height, 32, 8,
272  AOM_BORDER_IN_PIXELS)) {
273  die("Failed to allocate image.");
274  }
275 
276  printf("Using %s\n", aom_codec_iface_name(encoder->codec_interface()));
277 
278  res = aom_codec_enc_config_default(encoder->codec_interface(), &cfg, 0);
279  if (res) die_codec(&ecodec, "Failed to get default codec config.");
280 
281  cfg.g_w = info.frame_width;
282  cfg.g_h = info.frame_height;
283  cfg.g_timebase.num = info.time_base.numerator;
284  cfg.g_timebase.den = info.time_base.denominator;
285  cfg.rc_target_bitrate = bitrate;
286  cfg.g_lag_in_frames = 3;
287  cfg.g_bit_depth = AOM_BITS_8;
288 
289  flags |= (cfg.g_bit_depth > AOM_BITS_8 || !CONFIG_LOWBITDEPTH)
291  : 0;
292 
293  writer = aom_video_writer_open(outfile_arg, kContainerIVF, &info);
294  if (!writer) die("Failed to open %s for writing.", outfile_arg);
295 
296  if (!(infile = fopen(infile_arg, "rb")))
297  die("Failed to open %s for reading.", infile_arg);
298 
299  if (aom_codec_enc_init(&ecodec, encoder->codec_interface(), &cfg, flags))
300  die_codec(&ecodec, "Failed to initialize encoder");
301 
302  // Disable alt_ref.
304  die_codec(&ecodec, "Failed to set enable auto alt ref");
305 
306  if (test_decode) {
307  const AvxInterface *decoder = get_aom_decoder_by_name(codec_arg);
308  if (aom_codec_dec_init(&dcodec, decoder->codec_interface(), NULL, 0))
309  die_codec(&dcodec, "Failed to initialize decoder.");
310  }
311 
312  // Encode frames.
313  while (aom_img_read(&raw, infile)) {
314  if (limit && frame_in >= limit) break;
315  aom_image_t *frame_to_encode;
316 
317  if (!CONFIG_LOWBITDEPTH) {
318  // Need to allocate larger buffer to use hbd internal.
319  int input_shift = 0;
320  if (!allocated_raw_shift) {
321  aom_img_alloc(&raw_shift, raw_fmt | AOM_IMG_FMT_HIGHBITDEPTH,
322  info.frame_width, info.frame_height, 32);
323  allocated_raw_shift = 1;
324  }
325  aom_img_upshift(&raw_shift, &raw, input_shift);
326  frame_to_encode = &raw_shift;
327  } else {
328  frame_to_encode = &raw;
329  }
330 
331  if (update_frame_num > 1 && frame_out + 1 == update_frame_num) {
332  av1_ref_frame_t ref;
333  ref.idx = 0;
334  ref.use_external_ref = 0;
335  ref.img = ext_ref;
336  // Set reference frame in encoder.
337  if (aom_codec_control(&ecodec, AV1_SET_REFERENCE, &ref))
338  die_codec(&ecodec, "Failed to set encoder reference frame");
339  printf(" <SET_REF>");
340 
341  // If set_reference in decoder is commented out, the enc/dec mismatch
342  // would be seen.
343  if (test_decode) {
344  ref.use_external_ref = 1;
345  if (aom_codec_control(&dcodec, AV1_SET_REFERENCE, &ref))
346  die_codec(&dcodec, "Failed to set decoder reference frame");
347  }
348  }
349 
350  encode_frame(&ecodec, frame_to_encode, frame_in, writer, test_decode,
351  &dcodec, &frame_out, &mismatch_seen, &ext_ref);
352  frame_in++;
353  if (mismatch_seen) break;
354  }
355 
356  // Flush encoder.
357  if (!mismatch_seen)
358  while (encode_frame(&ecodec, NULL, frame_in, writer, test_decode, &dcodec,
359  &frame_out, &mismatch_seen, NULL)) {
360  }
361 
362  printf("\n");
363  fclose(infile);
364  printf("Processed %d frames.\n", frame_out);
365 
366  if (test_decode) {
367  if (!mismatch_seen)
368  printf("Encoder/decoder results are matching.\n");
369  else
370  printf("Encoder/decoder results are NOT matching.\n");
371  }
372 
373  if (test_decode)
374  if (aom_codec_destroy(&dcodec))
375  die_codec(&dcodec, "Failed to destroy decoder");
376 
377  if (allocated_raw_shift) aom_img_free(&raw_shift);
378  aom_img_free(&ext_ref);
379  aom_img_free(&raw);
380  if (aom_codec_destroy(&ecodec))
381  die_codec(&ecodec, "Failed to destroy encoder.");
382 
383  aom_video_writer_close(writer);
384 
385  return EXIT_SUCCESS;
386 }
Operation completed without error.
Definition: aom_codec.h:103
unsigned int d_h
Definition: aom_image.h:157
unsigned int g_w
Width of the frame.
Definition: aom_encoder.h:276
unsigned int rc_target_bitrate
Target data rate.
Definition: aom_encoder.h:491
Describes the encoder algorithm interface to applications.
#define aom_codec_enc_init(ctx, iface, cfg, flags)
Convenience macro for aom_codec_enc_init_ver()
Definition: aom_encoder.h:774
Encoder configuration structure.
Definition: aom_encoder.h:237
int idx
Definition: aom.h:109
enum aom_img_fmt aom_img_fmt_t
List of supported image formats.
Provides definitions for using AOM or AV1 encoder algorithm within the aom Codec Interface.
Codec context structure.
Definition: aom_codec.h:204
#define AOM_FRAME_IS_KEY
Definition: aom_encoder.h:104
#define AOM_IMG_FMT_HIGHBITDEPTH
Definition: aom_image.h:38
Describes the decoder algorithm interface to applications.
aom_codec_err_t aom_codec_enc_config_default(aom_codec_iface_t *iface, aom_codec_enc_cfg_t *cfg, unsigned int reserved)
Get a default configuration.
Image Descriptor.
Definition: aom_image.h:141
aom_image_t * aom_img_alloc(aom_image_t *img, aom_img_fmt_t fmt, unsigned int d_w, unsigned int d_h, unsigned int align)
Open a descriptor, allocating storage for the underlying image.
aom_codec_err_t aom_codec_decode(aom_codec_ctx_t *ctx, const uint8_t *data, size_t data_sz, void *user_priv)
Decode data.
const aom_codec_cx_pkt_t * aom_codec_get_cx_data(aom_codec_ctx_t *ctx, aom_codec_iter_t *iter)
Encoded data iterator.
aom_codec_err_t aom_codec_encode(aom_codec_ctx_t *ctx, const aom_image_t *img, aom_codec_pts_t pts, unsigned long duration, aom_enc_frame_flags_t flags)
Encode a frame.
Definition: aom_codec.h:225
#define aom_codec_dec_init(ctx, iface, cfg, flags)
Convenience macro for aom_codec_dec_init_ver()
Definition: aom_decoder.h:142
Definition: aom.h:60
const char * aom_codec_iface_name(aom_codec_iface_t *iface)
Return the name for a given interface.
struct aom_rational g_timebase
Stream timebase units.
Definition: aom_encoder.h:334
aom_codec_err_t aom_codec_destroy(aom_codec_ctx_t *ctx)
Destroy a codec instance.
aom_image_t * aom_img_alloc_with_border(aom_image_t *img, aom_img_fmt_t fmt, unsigned int d_w, unsigned int d_h, unsigned int align, unsigned int size_align, unsigned int border)
Open a descriptor, allocating storage for the underlying image with a border.
enum aom_codec_cx_pkt_kind kind
Definition: aom_encoder.h:148
void aom_img_free(aom_image_t *img)
Close an image descriptor.
struct aom_codec_cx_pkt::@1::@2 frame
Definition: aom_encoder.h:135
#define AOM_FRAME_IS_FRAGMENT
this is a fragment of the encoded frame
Definition: aom_encoder.h:111
#define aom_codec_control(ctx, id, data)
aom_codec_control wrapper macro
Definition: aom_codec.h:423
#define AOM_CODEC_USE_HIGHBITDEPTH
Definition: aom_encoder.h:78
Definition: aom.h:65
AV1 specific reference frame data struct.
Definition: aom.h:108
Codec control function to enable automatic set and use alf frames.
Definition: aomcx.h:190
const void * aom_codec_iter_t
Iterator.
Definition: aom_codec.h:194
int use_external_ref
Definition: aom.h:110
aom_codec_err_t
Algorithm return codes.
Definition: aom_codec.h:101
int den
Definition: aom_encoder.h:189
aom_image_t img
Definition: aom.h:111
Encoder output packet.
Definition: aom_encoder.h:147
int num
Definition: aom_encoder.h:188
Definition: aom_image.h:45
unsigned int g_lag_in_frames
Allow lagged encoding.
Definition: aom_encoder.h:363
union aom_codec_cx_pkt::@1 data
aom_bit_depth_t g_bit_depth
Bit-depth of the codec.
Definition: aom_encoder.h:312
unsigned int d_w
Definition: aom_image.h:156
unsigned int g_h
Height of the frame.
Definition: aom_encoder.h:285
aom_img_fmt_t fmt
Definition: aom_image.h:142