diff --git a/scripts/test-live-media.ts b/scripts/test-live-media.ts index c7e539f7296f..2b1c362d606e 100644 --- a/scripts/test-live-media.ts +++ b/scripts/test-live-media.ts @@ -256,6 +256,26 @@ function hasExplicitProviderSelection(options: CliOptions): boolean { return options.globalProviders !== null || Object.keys(options.suiteProviders).length > 0; } +function hasExplicitProviderSelectionForSuite(options: CliOptions, suiteId: MediaSuiteId): boolean { + if (Object.hasOwn(options.suiteProviders, suiteId)) { + return true; + } + if (!options.globalProviders) { + return false; + } + return MEDIA_SUITES[suiteId].providers.some((provider) => options.globalProviders?.has(provider)); +} + +export function findSkippedExplicitProviderSelections( + options: CliOptions, + plan: SuiteRunPlan[], +): SuiteRunPlan[] { + return plan.filter( + (entry) => + entry.providers.length === 0 && hasExplicitProviderSelectionForSuite(options, entry.suite.id), + ); +} + function selectProviders(params: { suite: MediaSuiteConfig; globalProviders: Set | null; @@ -396,6 +416,13 @@ export async function runCli(argv: string[]): Promise { `[live:media] skip ${entry.suite.id}: ${entry.skippedReason ?? "no providers selected"}`, ); } + const skippedExplicit = findSkippedExplicitProviderSelections(options, plan); + if (skippedExplicit.length > 0) { + console.error( + `[live:media] no runnable providers matched explicit provider selection for: ${skippedExplicit.map((entry) => entry.suite.id).join(", ")}`, + ); + return 1; + } if (runnable.length === 0) { console.log("[live:media] nothing to run"); if (hasExplicitProviderSelection(options)) { diff --git a/test/scripts/test-live-media.test.ts b/test/scripts/test-live-media.test.ts index c3bddef60c03..6316f3a1e1f1 100644 --- a/test/scripts/test-live-media.test.ts +++ b/test/scripts/test-live-media.test.ts @@ -1,6 +1,10 @@ // Test Live Media tests cover test live media script behavior. import { describe, expect, it } from "vitest"; -import { parseArgs } from "../../scripts/test-live-media.ts"; +import { + MEDIA_SUITES, + findSkippedExplicitProviderSelections, + parseArgs, +} from "../../scripts/test-live-media.ts"; describe("scripts/test-live-media", () => { it("rejects unknown global providers for the selected suites", () => { @@ -39,4 +43,40 @@ describe("scripts/test-live-media", () => { passthroughArgs: ["--project", "tooling", "-t", "media-smoke"], }); }); + + it("fails explicit suite selections that auth filtering would skip", () => { + const options = parseArgs([ + "image", + "music", + "--image-providers", + "openai", + "--music-providers", + "minimax", + ]); + const skipped = findSkippedExplicitProviderSelections(options, [ + { suite: MEDIA_SUITES.image, providers: ["openai"] }, + { + suite: MEDIA_SUITES.music, + providers: [], + skippedReason: "no providers with usable auth", + }, + ]); + + expect(skipped.map((entry) => entry.suite.id)).toEqual(["music"]); + }); + + it("does not fail global provider filters for suites without provider overlap", () => { + const options = parseArgs(["image", "music", "video", "--providers", "openai"]); + const skipped = findSkippedExplicitProviderSelections(options, [ + { suite: MEDIA_SUITES.image, providers: ["openai"] }, + { + suite: MEDIA_SUITES.music, + providers: [], + skippedReason: "no providers selected", + }, + { suite: MEDIA_SUITES.video, providers: ["openai"] }, + ]); + + expect(skipped).toEqual([]); + }); });