avformat/os_support: fix return value of win32_rename

The return value of MoveFileExW was not being correctly interpreted,
see https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-movefileexw.

On Windows a failed rename over the file: protocol now surfaces as failed to rename file %s to %s:
Operation not permitted (ff_rename, libavformat/avio.c:867) plus an AVERROR(EPERM) return, where previously the muxer
reported success and the user was left with a missing or stale output file and no diagnostic. This affects the
write-to-temp-then-rename paths in hlsenc, dashenc, hdsenc, smoothstreamingenc, segment, and img2enc (e.g. HLS/DASH
playlist updates, -write_temp_file, segment list finalization).

Also, added unit tests to exercise the rename function.

Signed-off-by: Christopher Decker <chris.decker08@gmail.com>

libavformat/Makefile

@@ -781,6 +781,7 @@ SKIPHEADERS-$(CONFIG_NETWORK)            += network.h rtsp.h
 
 TESTPROGS = id3v2                                                       \
             mkdir                                                       \
+            rename                                                      \
             seek                                                        \
             url                                                         \
             seek_utils

libavformat/os_support.h

@@ -279,7 +279,7 @@ static inline int win32_rename(const char *src_utf8, const char *dest_utf8)
         goto fallback;
     }
 
-    ret = MoveFileExW(src_w, dest_w, MOVEFILE_REPLACE_EXISTING);
+    ret = (MoveFileExW(src_w, dest_w, MOVEFILE_REPLACE_EXISTING) == 0) ? -1 : 0;
     av_free(src_w);
     av_free(dest_w);
     // Lacking proper mapping from GetLastError() error codes to errno codes
@@ -290,7 +290,7 @@ static inline int win32_rename(const char *src_utf8, const char *dest_utf8)
 fallback:
     /* filename may be be in CP_ACP */
 #if !HAVE_UWP
-    ret = MoveFileExA(src_utf8, dest_utf8, MOVEFILE_REPLACE_EXISTING);
+    ret = (MoveFileExA(src_utf8, dest_utf8, MOVEFILE_REPLACE_EXISTING) == 0) ? -1 : 0;
     if (ret)
         errno = EPERM;
 #else

libavformat/tests/rename.c

@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2026 Christopher Decker
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "config.h"
+
+#include <stdio.h>
+
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include "libavutil/random_seed.h"
+
+#include "libavformat/os_support.h"
+
+static int create_file(const char *path)
+{
+    FILE *f = fopen(path, "wb");
+    if (!f)
+        return -1;
+    fputs("ffmpeg rename test\n", f);
+    fclose(f);
+    return 0;
+}
+
+static int file_exists(const char *path)
+{
+    FILE *f = fopen(path, "rb");
+    if (!f)
+        return 0;
+    fclose(f);
+    return 1;
+}
+
+int main(void)
+{
+    char src[64];
+    char dst[64];
+    unsigned seed = av_get_random_seed();
+    int ret = 0;
+
+    snprintf(src, sizeof(src), "ff-rename-test-%08x.src", seed);
+    snprintf(dst, sizeof(dst), "ff-rename-test-%08x.dst", seed);
+
+    if (create_file(src) < 0) {
+        perror("create src");
+        return 1;
+    }
+
+    /* rename() must follow POSIX semantics and return 0 on success. */
+    if (rename(src, dst) != 0) {
+        perror("rename");
+        ret = 1;
+        goto cleanup;
+    }
+
+    if (file_exists(src)) {
+        fprintf(stderr, "source still exists after rename\n");
+        ret = 1;
+        goto cleanup;
+    }
+
+    if (!file_exists(dst)) {
+        fprintf(stderr, "destination missing after rename\n");
+        ret = 1;
+        goto cleanup;
+    }
+
+    /* Renaming a nonexistent source must fail with a -1 return. */
+    if (rename(src, dst) != -1) {
+        fprintf(stderr, "rename of nonexistent source unexpectedly succeeded\n");
+        ret = 1;
+        goto cleanup;
+    }
+
+cleanup:
+    unlink(src);
+    unlink(dst);
+    return ret;
+}

tests/fate/libavformat.mak

@@ -7,6 +7,11 @@ fate-mkdir: libavformat/tests/mkdir$(EXESUF)
 fate-mkdir: CMD = run libavformat/tests/mkdir$(EXESUF)
 fate-mkdir: CMP = null
 
+FATE_LIBAVFORMAT += fate-rename
+fate-rename: libavformat/tests/rename$(EXESUF)
+fate-rename: CMD = run libavformat/tests/rename$(EXESUF)
+fate-rename: CMP = null
+
 FATE_LIBAVFORMAT-$(CONFIG_NETWORK) += fate-noproxy
 fate-noproxy: libavformat/tests/noproxy$(EXESUF)
 fate-noproxy: CMD = run libavformat/tests/noproxy$(EXESUF)