avutil/{buffer,refstruct}: annotate pooled memory for ASan and MSan

Poison the memory when it enters the pool and unpoison it when it is
handed out again or freed. Only buffers with av_buffer_default_free are
touched, since custom pool allocators may not be compatible with it.

Also mark reused memory undefined, but only where the pool never
promised zeros.

RefStruct entries with an entry free callback own allocations while
they rest in the pool, such as the AVFrame behind an AVContainerFifo
slot. LeakSanitizer does not follow pointers stored in poisoned memory
and would report those allocations as leaked while the pool is alive,
so such entries stay addressable.

Signed-off-by: Kacper Michajłow <kasper93@gmail.com>

libavutil/buffer.c

@@ -24,6 +24,7 @@
 #include "buffer_internal.h"
 #include "common.h"
 #include "mem.h"
+#include "sanitizer.h"
 #include "thread.h"
 
 static AVBufferRef *buffer_create(AVBuffer *buf, uint8_t *data, size_t size,
@@ -305,6 +306,8 @@ static void buffer_pool_flush(AVBufferPool *pool)
         BufferPoolEntry *buf = pool->pool;
         pool->pool = buf->next;
 
+        if (buf->free == av_buffer_default_free)
+            FF_ASAN_UNPOISON(buf->data, pool->size);
         buf->free(buf->opaque, buf->data);
         av_freep(&buf);
     }
@@ -347,6 +350,9 @@ static void pool_release_buffer(void *opaque, uint8_t *data)
     BufferPoolEntry *buf = opaque;
     AVBufferPool *pool = buf->pool;
 
+    if (buf->free == av_buffer_default_free)
+        FF_ASAN_POISON(buf->data, pool->size);
+
     ff_mutex_lock(&pool->mutex);
     buf->next = pool->pool;
     pool->pool = buf;
@@ -402,6 +408,14 @@ AVBufferRef *av_buffer_pool_get(AVBufferPool *pool)
             pool->pool = buf->next;
             buf->next = NULL;
             buf->buffer.flags_internal |= BUFFER_FLAG_NO_FREE;
+            if (buf->free == av_buffer_default_free) {
+                FF_ASAN_UNPOISON(buf->data, pool->size);
+                /* A pool that zeroes its buffers may be relied upon for never
+                 * written regions staying zero, so only mark a reused buffer
+                 * undefined when nothing was ever promised about its contents. */
+                if (!pool->alloc2 && pool->alloc == av_buffer_alloc)
+                    FF_MEM_UNDEFINED(buf->data, pool->size);
+            }
         }
     } else {
         ret = pool_alloc_buffer(pool);

libavutil/refstruct.c

@@ -21,6 +21,7 @@
 #include <string.h>
 
 #include "refstruct.h"
+#include "sanitizer.h"
 
 #include "avassert.h"
 #include "error.h"
@@ -215,6 +216,8 @@ static void pool_free(AVRefStructPool *pool)
 
 static void pool_free_entry(AVRefStructPool *pool, RefCount *ref)
 {
+    if (!pool->free_entry_cb)
+        FF_ASAN_UNPOISON(get_userdata(ref), pool->size);
     if (pool->free_entry_cb)
         pool->free_entry_cb(pool->opaque, get_userdata(ref));
     av_free(ref);
@@ -227,6 +230,11 @@ static void pool_return_entry(void *ref_)
 
     ff_mutex_lock(&pool->mutex);
     if (!pool->uninited) {
+        /* An entry with an entry free callback owns allocations while it
+         * rests in the pool. LeakSanitizer does not follow pointers stored
+         * in poisoned memory, so such entries stay addressable. */
+        if (!pool->free_entry_cb)
+            FF_ASAN_POISON(get_userdata(ref), pool->size);
         ref->opaque.nc = pool->available_entries;
         pool->available_entries = ref;
         ref = NULL;
@@ -258,6 +266,16 @@ static int refstruct_pool_get_ext(void *datap, AVRefStructPool *pool)
     if (pool->available_entries) {
         RefCount *ref = pool->available_entries;
         ret = get_userdata(ref);
+        if (!pool->free_entry_cb)
+            FF_ASAN_UNPOISON(ret, pool->size);
+        /* Entries are zeroed once at allocation and a user may rely on never
+         * written parts staying zero, so only a pool that never zeroes and
+         * keeps no state through callbacks gets its reused entries marked
+         * undefined. */
+        if ((pool->entry_flags & AV_REFSTRUCT_FLAG_NO_ZEROING) &&
+            !(pool->pool_flags & AV_REFSTRUCT_POOL_FLAG_ZERO_EVERY_TIME) &&
+            !pool->init_cb && !pool->reset_cb && !pool->free_entry_cb)
+            FF_MEM_UNDEFINED(ret, pool->size);
         pool->available_entries = ref->opaque.nc;
         ref->opaque.nc = pool;
         atomic_init(&ref->refcount, 1);

libavutil/sanitizer.h

@@ -22,14 +22,34 @@
 #include "config.h"
 
 #if defined(__has_feature)
+#if __has_feature(address_sanitizer)
+#define HAVE_ASAN 1
+#endif
 #if __has_feature(memory_sanitizer)
 #define HAVE_MSAN 1
 #endif
 #endif
+#if defined(__SANITIZE_ADDRESS__)
+#undef  HAVE_ASAN
+#define HAVE_ASAN 1
+#endif
+#ifndef HAVE_ASAN
+#define HAVE_ASAN 0
+#endif
 #ifndef HAVE_MSAN
 #define HAVE_MSAN 0
 #endif
 
+/* Mark allocated memory that nothing may touch until it is unpoisoned. */
+#if HAVE_ASAN
+#include <sanitizer/asan_interface.h>
+#define FF_ASAN_POISON(ptr, size)   __asan_poison_memory_region(ptr, size)
+#define FF_ASAN_UNPOISON(ptr, size) __asan_unpoison_memory_region(ptr, size)
+#else
+#define FF_ASAN_POISON(ptr, size)   ((void)(ptr), (void)(size))
+#define FF_ASAN_UNPOISON(ptr, size) ((void)(ptr), (void)(size))
+#endif
+
 /* Mark memory as uninitialized. */
 #if HAVE_MSAN
 #include <sanitizer/msan_interface.h>