diff --git a/builder/dockerfile/dispatchers.go b/builder/dockerfile/dispatchers.go
index d16da1b77b..9d5fc14cf7 100644
--- a/builder/dockerfile/dispatchers.go
+++ b/builder/dockerfile/dispatchers.go
@@ -438,12 +438,11 @@ func run(req dispatchRequest) error {
 	// these args are transparent so resulting image should be the same regardless of the value
 	if len(cmdBuildEnv) > 0 {
 		saveCmd = config.Cmd
-		tmpBuildEnv := make([]string, len(cmdBuildEnv))
-		copy(tmpBuildEnv, cmdBuildEnv)
-		for i, env := range tmpBuildEnv {
+		var tmpBuildEnv []string
+		for _, env := range cmdBuildEnv {
 			key := strings.SplitN(env, "=", 2)[0]
-			if req.builder.buildArgs.IsUnreferencedBuiltin(key) {
-				tmpBuildEnv = append(tmpBuildEnv[:i], tmpBuildEnv[i+1:]...)
+			if !req.builder.buildArgs.IsUnreferencedBuiltin(key) {
+				tmpBuildEnv = append(tmpBuildEnv, env)
 			}
 		}
 		sort.Strings(tmpBuildEnv)
diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go
index cd996750cd..ed6633682b 100644
--- a/integration-cli/docker_cli_build_test.go
+++ b/integration-cli/docker_cli_build_test.go
@@ -4355,7 +4355,8 @@ func (s *DockerSuite) TestBuildTimeArgHistoryExclusions(c *check.C) {
 		ARG %s
 		RUN echo "Testing Build Args!"`, envKey, explicitProxyKey)
 	buildImage(imgName,
-		cli.WithFlags("--build-arg", fmt.Sprintf("%s=%s", envKey, envVal),
+		cli.WithFlags("--build-arg", "https_proxy=https://proxy.example.com",
+			"--build-arg", fmt.Sprintf("%s=%s", envKey, envVal),
 			"--build-arg", fmt.Sprintf("%s=%s", explicitProxyKey, explicitProxyVal),
 			"--build-arg", proxy),
 		build.WithDockerfile(dockerfile),
@@ -4365,6 +4366,9 @@ func (s *DockerSuite) TestBuildTimeArgHistoryExclusions(c *check.C) {
 	if strings.Contains(out, proxy) {
 		c.Fatalf("failed to exclude proxy settings from history!")
 	}
+	if strings.Contains(out, "https_proxy") {
+		c.Fatalf("failed to exclude proxy settings from history!")
+	}
 	if !strings.Contains(out, fmt.Sprintf("%s=%s", envKey, envVal)) {
 		c.Fatalf("explicitly defined ARG %s is not in output", explicitProxyKey)
 	}