avcodec/jpegxl_parser: fix handling of special ISOBMFF box sizes

skip_boxes() treated a box of size 0 and an empty box (size equal to
the header size) as invalid. jpegxl_parse() then set ctx->next to -1,
which disables the search for the end of the image for the rest of the
stream, so all following images were merged into a single packet,
which is only output at EOF.

An empty box is valid and is now skipped. A box of size 0 extends to
the end of the file, so the rest of the stream belongs to the current
image; keep that behaviour, but explicitly. Boxes too large to track
are treated the same way, as before. A box smaller than its header is
invalid: log an error and end the current image at that box instead
of silently swallowing the rest of the stream.

Signed-off-by: Martin Mirchev <mmirchev@comp.nus.edu.sg>

libavcodec/jpegxl_parser.c

@@ -1393,12 +1393,15 @@ static int skip_boxes(JXLParseContext *ctx, const uint8_t *buf, int buf_size)
                 return AVERROR_BUFFER_TOO_SMALL;
             size = bytestream2_get_be64(&gb);
             head_size = 16;
+        } else if (!size) {
+            /* the box extends to the end of the file */
+            return END_NOT_FOUND;
         }
-        if (!size)
-            return AVERROR_INVALIDDATA;
         /* invalid ISOBMFF size */
-        if (size <= head_size || size > INT_MAX - ctx->skip)
+        if (size < head_size)
             return AVERROR_INVALIDDATA;
+        if (size > INT_MAX - ctx->skip)
+            return END_NOT_FOUND;
 
         ctx->skip += size;
         bytestream2_skip(&gb, size - head_size);
@@ -1512,12 +1515,19 @@ static int jpegxl_parse(AVCodecParserContext *s, AVCodecContext *avctx,
 
     if (ctx->container && ctx->next >= 0) {
         ret = skip_boxes(ctx, pbuf, pindex);
-        if (ret < 0) {
-            if (ret == AVERROR_INVALIDDATA)
-                ctx->next = -1;
+        if (ret == END_NOT_FOUND) {
+            /* the rest of the stream belongs to this image */
+            ctx->next = -1;
             goto flush;
+        } else if (ret == AVERROR_INVALIDDATA) {
+            /* end the image at the invalid box */
+            av_log(avctx, AV_LOG_ERROR, "Invalid ISOBMFF box size\n");
+            ctx->next = ctx->skip;
+        } else if (ret < 0) {
+            goto flush;
+        } else {
+            ctx->next = ret + ctx->skip;
         }
-        ctx->next = ret + ctx->skip;
     }
 
     if (ctx->next >= 0)