diff --git a/gateway/session_api_media.py b/gateway/session_api_media.py index 3281734c5f..b2131f2169 100644 --- a/gateway/session_api_media.py +++ b/gateway/session_api_media.py @@ -34,12 +34,15 @@ def commit_api_images(content): from gateway.platforms.base import get_inbound_media_max_bytes from gateway.session_ingress_media import capture_native_media, sniff_image_mime from hermes_state_runtime import RuntimeStoreError - images = [decoded for decoded in map(_data_url_bytes, _image_urls(content)) if decoded is not None] + # The bytes decide the committed type: SDKs often label JPEG bytes ``data:image/png`` and + # main forwarded those to the provider; only bytes that are no supported image are refused. + images = [(sniff_image_mime(decoded[1]), decoded[1]) + for decoded in map(_data_url_bytes, _image_urls(content)) if decoded is not None] # Validate the whole batch before one byte lands on disk: a refused request must not - # leave staged or committed bytes behind, and the declared type must be the real one. + # leave staged or committed bytes behind. limit = max(0, get_inbound_media_max_bytes()) if (len(images) > _IMAGE_LIMIT or (limit and sum(len(data) for _, data in images) > limit) - or any(sniff_image_mime(data) != mime for mime, data in images)): + or any(mime is None for mime, _ in images)): raise RuntimeStoreError('invalid_params') staged = [] for mime, data in images: diff --git a/tests/gateway/test_api_image_mislabelled_mime.py b/tests/gateway/test_api_image_mislabelled_mime.py new file mode 100644 index 0000000000..d96ebdb201 --- /dev/null +++ b/tests/gateway/test_api_image_mislabelled_mime.py @@ -0,0 +1,17 @@ +"""A data URL whose label disagrees with its image bytes (``data:image/png`` carrying JPEG, a +common SDK default) is admitted under the sniffed type, as main forwarded it to the provider.""" +import base64 +from pathlib import Path + +from gateway.session_api_turn import admit_api_turn + +JPEG = b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00\xff\xd9' + + +def test_mislabelled_image_commits_under_sniffed_type(api, owner): + url = 'data:image/png;base64,' + base64.b64encode(JPEG).decode() + parts = [{'type': 'text', 'text': 'look'}, {'type': 'image_url', 'image_url': {'url': url}}] + _, _, row = admit_api_turn(api, session_id='api-mislabel', user_message=parts, conversation_history=[]) + media = row['payload']['api_turn_v1']['media'] + assert len(media) == 1 and Path(media[0]['path']).read_bytes() == JPEG + assert Path(media[0]['path']).suffix == '.jpg'