E2e run

.github/actions/verify-metrics-snapshot/action.yaml

@@ -0,0 +1,66 @@
+# Copyright (c) 2023 The Jaeger Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+name: 'Verify Metric Snapshot and Upload Metrics'
+description: 'Upload or cache the metrics data after verification'
+inputs:
+  snapshot:
+    description: 'Path to the metric file'
+    required: true
+  artifact_key:
+    description: 'Artifact key used for uploading and fetching artifacts'
+    required: true
+  cache_key:
+    description: 'Cache key used for uploading and fetching the correct cached metric'
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - name: Upload Metrics Artifact
+      uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+      with:
+        name: ${{ inputs.artifact_key }}
+        path: ./.metrics/${{ inputs.snapshot }}
+        retention-days: 7
+
+    - name: Get v2 tag 
+      shell: bash
+      run: |
+        v2_tag=$(make echo-v2)
+        echo "v2_tag=$v2_tag" >> $GITHUB_ENV
+        echo "$v2_tag"
+
+    - name: Cache scraped metrics for tagged release for longer retention
+      id: tagged-metrics
+      if: "contains(github.event.head_commit.message, 'Release')"
+      uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57
+      with:
+        path: ./.metrics/${{ inputs.snapshot }}
+        key: ${{ env.v2_tag }}_${{ inputs.artifact_key }}
+
+    - name: Download the cached tagged metrics
+      id: cache-metrics
+      uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57
+      with:
+        path: ./.metrics/cached_${{ inputs.snapshot}}
+        key: ${{ env.v2_tag }}_${{ inputs.artifact_key }}
+
+    - name: Calculate diff between the snapshots 
+      id: diff-check
+      if: steps.cache-metrics.outputs.cache-hit == 'true'
+      shell: bash
+      run: |
+        python3 -m pip install prometheus-client
+        if ! python3 ./scripts/e2e/compare_metrics.py --file1 ./.metrics/${{ inputs.snapshot }} --file2 ./.metrics/cached_${{ inputs.snapshot}} --output ./.metrics/diff_${{ inputs.snapshot }}; then 
+          exit 1
+        fi
+
+    - name: Upload the diff artifact
+      if: steps.cache-metrics.outputs.cache-hit == 'true' && steps.diff-check.outcome == 'failure'
+      uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0
+      with:
+        name: diff_${{ inputs.artifact_key }}
+        path: ./.metrics/diff_${{ inputs.snapshot }}
+        retention-days: 7
+
+

.github/workflows/ci-e2e-all.yml

@@ -2,10 +2,12 @@ name: E2E Tests
 
 on:
   push:
-    branches: [main]
+    branches: [main, e2e_run]
+    tags:
+      - 'v*'
 
-  pull_request:
-    branches: [main]
+  # pull_request:
+  #   branches: [main]
 
 concurrency:
   group: combined-cit-${{ github.workflow }}-${{ (github.event.pull_request && github.event.pull_request.number) || github.ref || github.run_id }}
@@ -36,7 +38,3 @@ jobs:
   opensearch:
     uses: ./.github/workflows/ci-e2e-opensearch.yml
 
-
-
-
-

.github/workflows/ci-e2e-badger.yaml

@@ -26,6 +26,10 @@ jobs:
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
 
+    - name: Fetch git tags
+      run: |
+        git fetch --prune --unshallow --tags
+
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
@@ -41,6 +45,12 @@ jobs:
             ;;
         esac
 
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: ${{ matrix.version }} == "v2"
+      with:
+        snapshot: badger_metrics.txt
+        artifact_key: badger-metrics-${{ matrix.version }}.txt
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-cassandra.yml

@@ -39,6 +39,10 @@ jobs:
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
 
+    - name: Fetch git tags
+      run: |
+        git fetch --prune --unshallow --tags
+
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
@@ -48,9 +52,17 @@ jobs:
       run: bash scripts/e2e/cassandra.sh ${{ matrix.version.major }} ${{ matrix.version.schema }} ${{ matrix.jaeger-version }}
       env:
         SKIP_APPLY_SCHEMA: ${{ matrix.create-schema == 'auto' && true || false }}
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: ${{ matrix.jaeger-version  == 'v2'}}
+      with:
+        snapshot: cassandra_metrics.txt
+        artifact_key: cassandra_metrics_${{ matrix.version.major }}_${{ matrix.version.schema }}_${{ matrix.jaeger-version }}_${{ matrix.create-schema }}.txt
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:
         files: cover.out
         flags: cassandra-${{ matrix.version.major }}-${{ matrix.jaeger-version }}-${{ matrix.create-schema }}
+
+

.github/workflows/ci-e2e-elasticsearch.yml

@@ -58,6 +58,12 @@ jobs:
     - name: Run ${{ matrix.version.distribution }} integration tests
       id: test-execution
       run: bash scripts/e2e/elasticsearch.sh ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }}
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: ${{ matrix.version.jaeger }} == "v2"
+      with:
+        snapshot: elasticsearch_metrics.txt
+        artifact_key: elasticsearch-metrics-${{ matrix.version.major }}_${{ matrix.version.jaeger}}.txt
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov

.github/workflows/ci-e2e-grpc.yml

@@ -26,6 +26,10 @@ jobs:
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
 
+    - name: Fetch git tags
+      run: |
+        git fetch --prune --unshallow --tags
+
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
@@ -41,6 +45,12 @@ jobs:
             ;;
         esac
 
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: ${{ matrix.version }} == "v2"
+      with:
+        snapshot: grpc_metrics.txt
+        artifact_key: grpc-metrics-${{ matrix.version }}.txt
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-kafka.yml

@@ -28,13 +28,23 @@ jobs:
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
 
+    - name: Fetch git tags
+      run: |
+        git fetch --prune --unshallow --tags
+
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
 
     - name: Run kafka integration tests
       id: test-execution
       run: bash scripts/e2e/kafka.sh -j ${{ matrix.jaeger-version }} -v ${{ matrix.kafka-version }}
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: ${{ matrix.jaeger-version }} == "v2"
+      with:
+        snapshot: kafka_metrics.txt
+        artifact_key: kafka-metrics-${{ matrix.jaeger-version }}.txt
 
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov

.github/workflows/ci-e2e-memory.yaml

@@ -22,14 +22,23 @@ jobs:
 
     - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
 
+    - name: Fetch git tags
+      run: |
+        git fetch --prune --unshallow --tags
+
     - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
       with:
         go-version: 1.23.x
 
     - name: Run Memory storage integration tests
       run: |
         STORAGE=memory_v2 make jaeger-v2-storage-integration-test
-
+
+    - uses: ./.github/actions/verify-metrics-snapshot
+      with:
+        snapshot: memory_metrics.txt
+        artifact_key: memory-metrics.txt
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.github/workflows/ci-e2e-opensearch.yml

@@ -52,6 +52,12 @@ jobs:
       id: test-execution
       run: bash scripts/e2e/elasticsearch.sh ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }}
 
+    - uses: ./.github/actions/verify-metrics-snapshot
+      if: ${{ matrix.version.jaeger }} == "v2"
+      with:
+        snapshot: opensearch_metrics.txt
+        artifact_key: opensearch-metrics-${{ matrix.version.major }}.txt
+
     - name: Upload coverage to codecov
       uses: ./.github/actions/upload-codecov
       with:

.gitignore

@@ -50,3 +50,4 @@ sha256sum.combined.txt
 resource.syso
 .gocache
 test-results.json
+.metrics/

cmd/jaeger/internal/integration/e2e_integration.go

@@ -99,11 +99,32 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) {
 	require.NoError(t, err)
 
 	t.Cleanup(func() {
+		scrapeMetrics(t, storage)
 		require.NoError(t, s.TraceReader.(io.Closer).Close())
 		require.NoError(t, s.TraceWriter.(io.Closer).Close())
 	})
 }
 
+func scrapeMetrics(t *testing.T, storage string) {
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://localhost:8888/metrics", nil)
+	require.NoError(t, err)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	outputDir := "../../../../.metrics"
+	require.NoError(t, os.MkdirAll(outputDir, os.ModePerm))
+
+	metricsFile, err := os.Create(fmt.Sprintf("%s/%v_metrics.txt", outputDir, storage))
+	require.NoError(t, err)
+	defer metricsFile.Close()
+
+	_, err = io.Copy(metricsFile, resp.Body)
+	require.NoError(t, err)
+}
+
 func createStorageCleanerConfig(t *testing.T, configFile string, storage string) string {
 	data, err := os.ReadFile(configFile)
 	require.NoError(t, err)

cmd/jaeger/internal/integration/trace_writer.go

@@ -81,7 +81,7 @@ func (w *traceWriter) WriteTraces(ctx context.Context, td ptrace.Traces) error {
 			scope    ptrace.ScopeSpans
 			resource ptrace.ResourceSpans
 		)
-		
+
 		if spanCount == MaxChunkSize {
 			err = w.exporter.ConsumeTraces(ctx, currentChunk)
 			currentChunk = ptrace.NewTraces()

scripts/e2e/compare_metrics.py

@@ -0,0 +1,74 @@
+# Copyright (c) 2024 The Jaeger Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import sys
+from difflib import unified_diff
+from bisect import insort
+from prometheus_client.parser import text_string_to_metric_families
+
+def read_metric_file(file_path):
+    with open(file_path, 'r') as f:
+        return f.readlines()
+
+def parse_metrics(content):
+    metrics = []
+    for family in text_string_to_metric_families(content):
+        for sample in family.samples:
+            labels = dict(sample.labels)
+            #simply pop undesirable metric labels
+            labels.pop('service_instance_id',None)
+            label_pairs = sorted(labels.items(), key=lambda x: x[0])
+            label_str = ','.join(f'{k}="{v}"' for k,v in label_pairs)
+            metric = f"{family.name}{{{label_str}}}"
+            insort(metrics , metric)
+
+    return metrics
+
+
+def generate_diff(file1_content, file2_content):
+    if isinstance(file1_content, list):
+        file1_content = ''.join(file1_content)
+    if isinstance(file2_content, list):
+        file2_content = ''.join(file2_content)
+
+    metrics1 = parse_metrics(file1_content)
+    metrics2 = parse_metrics(file2_content)
+
+    diff = unified_diff(metrics1, metrics2,lineterm='',n=0)
+
+    return '\n'.join(diff)
+
+def write_diff_file(diff_lines, output_path):
+
+    with open(output_path, 'w') as f:
+        f.write(diff_lines)
+        f.write('\n')  # Add final newline
+        print(f"Diff file successfully written to: {output_path}")
+
+def main():
+    parser = argparse.ArgumentParser(description='Generate diff between two Jaeger metric files')
+    parser.add_argument('--file1', help='Path to first metric file')
+    parser.add_argument('--file2', help='Path to second metric file')
+    parser.add_argument('--output', '-o', default='metrics_diff.txt',
+                       help='Output diff file path (default: metrics_diff.txt)')
+
+    args = parser.parse_args()
+
+    # Read input files
+    file1_lines = read_metric_file(args.file1)
+    file2_lines = read_metric_file(args.file2)
+
+    # Generate diff
+    diff_lines = generate_diff(file1_lines, file2_lines, str(args.file1), str(args.file2))
+
+    # Check if there are any differences
+    if not diff_lines:
+        print("No differences found between the metric files.")
+        sys.exit(0)
+
+    # Write diff to output file
+    write_diff_file(diff_lines, args.output)
+
+if __name__ == '__main__':
+    main()