From 517ae20be8f8a2aef37d07ff2284986ace6a9c44 Mon Sep 17 00:00:00 2001 From: Cory Snider Date: Thu, 20 Nov 2025 16:48:53 -0500 Subject: [PATCH] .github: collect all the OTEL traces Jaeger does not make it easy to dump all the collected trace spans from all services at once. Switch to using the OpenTelemetry Collector with the OTLP File exporter which writes the traces straight to disk in a format that Jaeger UI can natively consume. Signed-off-by: Cory Snider --- .github/actions/setup-tracing/action.yml | 15 ++++++++--- .github/workflows/.test.yml | 11 ++++---- .github/workflows/.windows.yml | 33 ++++++++++++------------ .github/workflows/arm64.yml | 3 ++- otelcol-ci-config.yml | 24 +++++++++++++++++ 5 files changed, 60 insertions(+), 26 deletions(-) create mode 100644 otelcol-ci-config.yml diff --git a/.github/actions/setup-tracing/action.yml b/.github/actions/setup-tracing/action.yml index b387b52b9b..a59b5737f1 100644 --- a/.github/actions/setup-tracing/action.yml +++ b/.github/actions/setup-tracing/action.yml @@ -6,9 +6,18 @@ runs: steps: - run: | set -e - # Jaeger is set up on Windows through an inline run step. If you update Jaeger here, don't forget to update - # the version set in .github/workflows/.windows.yml. - docker run -d --net=host --name jaeger -e COLLECTOR_OTLP_ENABLED=true jaegertracing/all-in-one:1.46 + # The OTEL Collector is set up on Windows through an inline run step. If + # you update the collector here, don't forget to update the version set + # in .github/workflows/.windows.yml. + mkdir -p /tmp/reports + chmod 777 /tmp/reports + docker run -d --net=host --name otelcol \ + -v "$(pwd)/otelcol-ci-config.yml:/etc/otelcol-contrib/config.yaml" \ + -v "/tmp/reports:/data" \ + otel/opentelemetry-collector-contrib:0.140.0 \ + --config file:/etc/otelcol-contrib/config.yaml \ + --config "yaml:exporters::file::path: /data/otel-trace.jsonl" docker0_ip="$(ip -f inet addr show docker0 | grep -Po 'inet \K[\d.]+')" echo "OTEL_EXPORTER_OTLP_ENDPOINT=http://${docker0_ip}:4318" >> "${GITHUB_ENV}" + echo "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf" >> "${GITHUB_ENV}" shell: bash diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml index 653165b263..04cb353b5e 100644 --- a/.github/workflows/.test.yml +++ b/.github/workflows/.test.yml @@ -68,13 +68,12 @@ jobs: name: Prepare reports if: always() run: | + docker stop otelcol mkdir -p bundles /tmp/reports find bundles -path '*/root/*overlay2' -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print | xargs sudo tar -czf /tmp/reports.tar.gz tar -xzf /tmp/reports.tar.gz -C /tmp/reports sudo chown -R $(id -u):$(id -g) /tmp/reports tree -nh /tmp/reports - - curl -sSLf localhost:16686/api/traces?service=integration-test-client > /tmp/reports/jaeger-trace.json - name: Test daemon logs if: always() @@ -228,13 +227,13 @@ jobs: reportsPath="/tmp/reports/$reportsName" echo "TESTREPORTS_NAME=$reportsName" >> $GITHUB_ENV + docker stop otelcol mkdir -p bundles $reportsPath find bundles -path '*/root/*overlay2' -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print | xargs sudo tar -czf /tmp/reports.tar.gz tar -xzf /tmp/reports.tar.gz -C $reportsPath + mv /tmp/reports/otel-trace*.jsonl $reportsPath/ sudo chown -R $(id -u):$(id -g) $reportsPath tree -nh $reportsPath - - curl -sSLf localhost:16686/api/traces?service=integration-test-client > $reportsPath/jaeger-trace.json - name: Send to Codecov uses: codecov/codecov-action@v4 @@ -443,14 +442,14 @@ jobs: reportsPath=/tmp/reports/$reportsName echo "TESTREPORTS_NAME=$reportsName" >> $GITHUB_ENV + docker stop otelcol mkdir -p bundles $reportsPath echo "${{ matrix.test }}" | tr -s '|' '\n' | tee -a "$reportsPath/tests.txt" find bundles -path '*/root/*overlay2' -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print | xargs sudo tar -czf /tmp/reports.tar.gz tar -xzf /tmp/reports.tar.gz -C $reportsPath + mv /tmp/reports/otel-trace*.jsonl $reportsPath/ sudo chown -R $(id -u):$(id -g) $reportsPath tree -nh $reportsPath - - curl -sSLf localhost:16686/api/traces?service=integration-test-client > $reportsPath/jaeger-trace.json - name: Send to Codecov uses: codecov/codecov-action@v4 diff --git a/.github/workflows/.windows.yml b/.github/workflows/.windows.yml index a1423f11ad..59c55c890b 100644 --- a/.github/workflows/.windows.yml +++ b/.github/workflows/.windows.yml @@ -274,14 +274,17 @@ jobs: go-version: ${{ env.GO_VERSION }} cache: false - - name: Set up Jaeger + name: Set up OpenTelemetry Collector run: | - # Jaeger is set up on Linux through the setup-tracing action. If you update Jaeger here, don't forget to + # The collectors is set up on Linux through the setup-tracing action. If you update the collector here, don't forget to # update the version set in .github/actions/setup-tracing/action.yml. - Invoke-WebRequest -Uri "https://github.com/jaegertracing/jaeger/releases/download/v1.46.0/jaeger-1.46.0-windows-amd64.tar.gz" -OutFile ".\jaeger-1.46.0-windows-amd64.tar.gz" - tar -zxvf ".\jaeger-1.46.0-windows-amd64.tar.gz" - Start-Process '.\jaeger-1.46.0-windows-amd64\jaeger-all-in-one.exe' - echo "OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + New-Item -ItemType Directory -Force -Path bundles -ErrorAction Continue + Start-Process "msiexec" -ArgumentList "/i", + "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v0.140.0/otelcol_0.140.0_windows_x64.msi", + "/qn", "/l*v", "$(Join-Path (Get-Location) "bundles/otelcol-install.log")", + "COLLECTOR_SVC_ARGS=`"--config=`"`"file:$(Join-Path (Get-Location) "otelcol-ci-config.yml")`"`" --config=`"`"yaml:exporters::file::path: $(Join-Path (Get-Location) "bundles/otel-trace.jsonl")`"`"`"" ` + -NoNewWindow -Wait + @("OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:4318", "OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf") | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append shell: pwsh - name: Env @@ -296,7 +299,7 @@ jobs: - name: Init run: | - New-Item -ItemType "directory" -Path "bundles" + New-Item -ItemType "directory" -Path "bundles" -ErrorAction SilentlyContinue If ("${{ inputs.os }}" -eq "windows-2025") { echo "WINDOWS_BASE_IMAGE_TAG=${{ env.WINDOWS_BASE_TAG_2025 }}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append } ElseIf ("${{ inputs.os }}" -eq "windows-2022") { @@ -342,14 +345,14 @@ jobs: "--exec-root=$env:TEMP\moby-exec", ` "--pidfile=$env:TEMP\docker.pid", ` "--register-service" + # Make the env-var visible to the service-managed dockerd, as there's no CLI flag for this option. + $dockerEnviron = @("DOCKER_MIN_API_VERSION=1.24") + $dockerEnviron += @(Get-Item Env:\OTEL_* | ForEach-Object { "$($_.Name)=$($_.Value)" }) If ("${{ inputs.storage }}" -eq "graphdriver") { - # Make the env-var visible to the service-managed dockerd, as there's no CLI flag for this option. - & reg add "HKLM\SYSTEM\CurrentControlSet\Services\docker" /v Environment /t REG_MULTI_SZ /s '@' /d "DOCKER_MIN_API_VERSION=1.24@TEST_INTEGRATION_USE_GRAPHDRIVER=1" + $dockerEnviron += @("TEST_INTEGRATION_USE_GRAPHDRIVER=1") echo "TEST_INTEGRATION_USE_GRAPHDRIVER=1" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append - } Else { - # Make the env-var visible to the service-managed dockerd, as there's no CLI flag for this option. - & reg add "HKLM\SYSTEM\CurrentControlSet\Services\docker" /v Environment /t REG_MULTI_SZ /s '@' /d DOCKER_MIN_API_VERSION=1.24 } + New-ItemProperty -Name "Environment" -Path "HKLM:\SYSTEM\CurrentControlSet\Services\docker" -PropertyType MultiString -Value $dockerEnviron Write-Host "Starting service" Start-Service -Name docker Write-Host "Service started successfully!" @@ -459,12 +462,10 @@ jobs: ForEach-Object {"$($_.TimeCreated.ToUniversalTime().ToString("o")) [$($_.LevelDisplayName)] $($_.Message)"} | Tee-Object -file ".\bundles\daemon.log" - - name: Download Jaeger traces + name: Stop OpenTelemetry Collector if: always() run: | - Invoke-WebRequest ` - -Uri "http://127.0.0.1:16686/api/traces?service=integration-test-client" ` - -OutFile ".\bundles\jaeger-trace.json" + (Stop-Service -DisplayName "OpenTelemetry Collector" -PassThru).WaitForStatus('Stopped', (New-TimeSpan -Seconds 30)) - name: Upload reports if: always() diff --git a/.github/workflows/arm64.yml b/.github/workflows/arm64.yml index ee35442328..8c5a783884 100644 --- a/.github/workflows/arm64.yml +++ b/.github/workflows/arm64.yml @@ -226,13 +226,14 @@ jobs: name: Prepare reports if: always() run: | + docker stop otelcol reportsPath="/tmp/reports/arm64-graphdriver" mkdir -p bundles $reportsPath find bundles -path '*/root/*overlay2' -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print | xargs sudo tar -czf /tmp/reports.tar.gz tar -xzf /tmp/reports.tar.gz -C $reportsPath + mv /tmp/reports/otel-trace*.jsonl $reportsPath/ sudo chown -R $(id -u):$(id -g) $reportsPath tree -nh $reportsPath - curl -sSLf localhost:16686/api/traces?service=integration-test-client > $reportsPath/jaeger-trace.json - name: Send to Codecov uses: codecov/codecov-action@v4 diff --git a/otelcol-ci-config.yml b/otelcol-ci-config.yml new file mode 100644 index 0000000000..301bfec020 --- /dev/null +++ b/otelcol-ci-config.yml @@ -0,0 +1,24 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + file: + flush_interval: 1s + rotation: + max_megabytes: 18 # Jaeger will reject file uploads larger than 20MB by default + # path: /data/metrics.json + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [file]