From 93259387b52b6a10c45a4634d110af138aecf597 Mon Sep 17 00:00:00 2001 From: chahat sagar <109112505+chahatsagarmain@users.noreply.github.com> Date: Mon, 6 Jan 2025 00:34:52 +0530 Subject: [PATCH] E2e run (#2) * script to scrape metrics Signed-off-by: chahatsagarmain * minor change Signed-off-by: chahatsagarmain * fix dir path Signed-off-by: chahatsagarmain * add metric upload to each test and changes in the scrape script Signed-off-by: chahatsagarmain * minor changes Signed-off-by: chahatsagarmain * changes Signed-off-by: chahatsagarmain * shell lint Signed-off-by: chahatsagarmain * minor changes Signed-off-by: chahatsagarmain * reviewed changes Signed-off-by: chahatsagarmain * added cache save Signed-off-by: chahatsagarmain * fixes Signed-off-by: chahatsagarmain * add schema in key Signed-off-by: chahatsagarmain * test ci Signed-off-by: chahatsagarmain * test ci Signed-off-by: chahatsagarmain * minor changes Signed-off-by: chahatsagarmain * test ci with fixed expression Signed-off-by: chahatsagarmain * shared action and a happy new yar Signed-off-by: chahatsagarmain * minor fixes Signed-off-by: chahatsagarmain * shared action Signed-off-by: chahatsagarmain * add shell property Signed-off-by: chahatsagarmain * compare metrics Signed-off-by: chahatsagarmain * fix path Signed-off-by: chahatsagarmain * fix compare Signed-off-by: chahatsagarmain * minor changes Signed-off-by: chahatsagarmain * compare metrics using python libraries Signed-off-by: chahatsagarmain * check path Signed-off-by: chahatsagarmain * resolved comments Signed-off-by: chahatsagarmain * use simple unified diff Signed-off-by: chahatsagarmain * exit cases Signed-off-by: chahatsagarmain * simplified action Signed-off-by: chahatsagarmain * exit case Signed-off-by: chahatsagarmain * fetch tags after checkout Signed-off-by: chahatsagarmain * test workflow Signed-off-by: chahatsagarmain * test Signed-off-by: chahatsagarmain * cache fix Signed-off-by: chahatsagarmain * test ci Signed-off-by: chahatsagarmain * test ci 2 Signed-off-by: chahatsagarmain * debug Signed-off-by: chahatsagarmain * test ci 3 Signed-off-by: chahatsagarmain * e2e_run Signed-off-by: chahatsagarmain * e2e_run 2 Signed-off-by: chahatsagarmain --------- Signed-off-by: chahatsagarmain Signed-off-by: chahat sagar <109112505+chahatsagarmain@users.noreply.github.com> --- .../verify-metrics-snapshot/action.yaml | 66 +++++++++++++++++ .github/workflows/ci-e2e-all.yml | 12 ++- .github/workflows/ci-e2e-badger.yaml | 10 +++ .github/workflows/ci-e2e-cassandra.yml | 12 +++ .github/workflows/ci-e2e-elasticsearch.yml | 6 ++ .github/workflows/ci-e2e-grpc.yml | 10 +++ .github/workflows/ci-e2e-kafka.yml | 10 +++ .github/workflows/ci-e2e-memory.yaml | 11 ++- .github/workflows/ci-e2e-opensearch.yml | 6 ++ .gitignore | 1 + .../internal/integration/e2e_integration.go | 21 ++++++ .../internal/integration/trace_writer.go | 2 +- scripts/e2e/compare_metrics.py | 74 +++++++++++++++++++ 13 files changed, 232 insertions(+), 9 deletions(-) create mode 100644 .github/actions/verify-metrics-snapshot/action.yaml create mode 100644 scripts/e2e/compare_metrics.py diff --git a/.github/actions/verify-metrics-snapshot/action.yaml b/.github/actions/verify-metrics-snapshot/action.yaml new file mode 100644 index 00000000000..54ca1a43929 --- /dev/null +++ b/.github/actions/verify-metrics-snapshot/action.yaml @@ -0,0 +1,66 @@ +# Copyright (c) 2023 The Jaeger Authors. +# SPDX-License-Identifier: Apache-2.0 + +name: 'Verify Metric Snapshot and Upload Metrics' +description: 'Upload or cache the metrics data after verification' +inputs: + snapshot: + description: 'Path to the metric file' + required: true + artifact_key: + description: 'Artifact key used for uploading and fetching artifacts' + required: true + cache_key: + description: 'Cache key used for uploading and fetching the correct cached metric' + required: true +runs: + using: 'composite' + steps: + - name: Upload Metrics Artifact + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: ${{ inputs.artifact_key }} + path: ./.metrics/${{ inputs.snapshot }} + retention-days: 7 + + - name: Cache scraped metrics for tagged release for longer retention + id: tagged-metrics + if: contains(github.ref,'tags') + uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 + with: + path: ./.metrics/${{ inputs.snapshot }} + key: ${{ github.ref_name }}_${{ inputs.artifact_key }} + + - name: Get v2 tag + shell: bash + run: | + v2_tag=$(make echo-v2) + echo "v2_tag=$v2_tag" >> $GITHUB_ENV + echo "$v2_tag" + + - name: Download the cached tagged metrics + id: cache-metrics + uses: actions/cache/restore@1bd1e32a3bdc45362d1e726936510720a7c30a57 + with: + path: ./.metrics/cached_${{ inputs.snapshot}} + key: ${{ env.v2_tag }}_${{ inputs.artifact_key }} + + - name: Calculate diff between the snapshots + id: diff-check + if: steps.cache-metrics.outputs.cache-hit == 'true' + shell: bash + run: | + python3 -m pip install prometheus-client + if ! python3 ./scripts/e2e/compare_metrics.py --file1 ./.metrics/${{ inputs.snapshot }} --file2 ./.metrics/cached_${{ inputs.snapshot}} --output ./.metrics/diff_${{ inputs.snapshot }}; then + exit 1 + fi + + - name: Upload the diff artifact + if: steps.cache-metrics.outputs.cache-hit == 'true' && steps.diff-check.outcome == 'failure' + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 + with: + name: diff_${{ inputs.artifact_key }} + path: ./.metrics/diff_${{ inputs.snapshot }} + retention-days: 7 + + \ No newline at end of file diff --git a/.github/workflows/ci-e2e-all.yml b/.github/workflows/ci-e2e-all.yml index fb41a17b5c5..41a56e22e78 100644 --- a/.github/workflows/ci-e2e-all.yml +++ b/.github/workflows/ci-e2e-all.yml @@ -2,10 +2,12 @@ name: E2E Tests on: push: - branches: [main] + branches: [main,e2e_run] + tags: + - 'v*' - pull_request: - branches: [main] + # pull_request: + # branches: [main] concurrency: group: combined-cit-${{ github.workflow }}-${{ (github.event.pull_request && github.event.pull_request.number) || github.ref || github.run_id }} @@ -36,7 +38,3 @@ jobs: opensearch: uses: ./.github/workflows/ci-e2e-opensearch.yml - - - - diff --git a/.github/workflows/ci-e2e-badger.yaml b/.github/workflows/ci-e2e-badger.yaml index 808b73fc6ce..260923f1d23 100644 --- a/.github/workflows/ci-e2e-badger.yaml +++ b/.github/workflows/ci-e2e-badger.yaml @@ -26,6 +26,10 @@ jobs: - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Fetch git tags + run: | + git fetch --prune --unshallow --tags + - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: go-version: 1.23.x @@ -41,6 +45,12 @@ jobs: ;; esac + - uses: ./.github/actions/verify-metrics-snapshot + if: ${{ matrix.version }} == "v2" + with: + snapshot: badger_metrics.txt + artifact_key: badger-metrics-${{ matrix.version }}.txt + - name: Upload coverage to codecov uses: ./.github/actions/upload-codecov with: diff --git a/.github/workflows/ci-e2e-cassandra.yml b/.github/workflows/ci-e2e-cassandra.yml index 1d866828736..8d7c81ce898 100644 --- a/.github/workflows/ci-e2e-cassandra.yml +++ b/.github/workflows/ci-e2e-cassandra.yml @@ -39,6 +39,10 @@ jobs: - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Fetch git tags + run: | + git fetch --prune --unshallow --tags + - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: go-version: 1.23.x @@ -48,9 +52,17 @@ jobs: run: bash scripts/e2e/cassandra.sh ${{ matrix.version.major }} ${{ matrix.version.schema }} ${{ matrix.jaeger-version }} env: SKIP_APPLY_SCHEMA: ${{ matrix.create-schema == 'auto' && true || false }} + + - uses: ./.github/actions/verify-metrics-snapshot + if: ${{ matrix.jaeger-version == 'v2'}} + with: + snapshot: cassandra_metrics.txt + artifact_key: cassandra_metrics_${{ matrix.version.major }}_${{ matrix.version.schema }}_${{ matrix.jaeger-version }}_${{ matrix.create-schema }}.txt - name: Upload coverage to codecov uses: ./.github/actions/upload-codecov with: files: cover.out flags: cassandra-${{ matrix.version.major }}-${{ matrix.jaeger-version }}-${{ matrix.create-schema }} + + diff --git a/.github/workflows/ci-e2e-elasticsearch.yml b/.github/workflows/ci-e2e-elasticsearch.yml index d6d90ed55f3..432533ab786 100644 --- a/.github/workflows/ci-e2e-elasticsearch.yml +++ b/.github/workflows/ci-e2e-elasticsearch.yml @@ -58,6 +58,12 @@ jobs: - name: Run ${{ matrix.version.distribution }} integration tests id: test-execution run: bash scripts/e2e/elasticsearch.sh ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }} + + - uses: ./.github/actions/verify-metrics-snapshot + if: ${{ matrix.version.jaeger }} == "v2" + with: + snapshot: elasticsearch_metrics.txt + artifact_key: elasticsearch-metrics-${{ matrix.version.major }}_${{ matrix.version.jaeger}}.txt - name: Upload coverage to codecov uses: ./.github/actions/upload-codecov diff --git a/.github/workflows/ci-e2e-grpc.yml b/.github/workflows/ci-e2e-grpc.yml index 4126c10b352..49119f06b4b 100644 --- a/.github/workflows/ci-e2e-grpc.yml +++ b/.github/workflows/ci-e2e-grpc.yml @@ -26,6 +26,10 @@ jobs: - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Fetch git tags + run: | + git fetch --prune --unshallow --tags + - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: go-version: 1.23.x @@ -41,6 +45,12 @@ jobs: ;; esac + - uses: ./.github/actions/verify-metrics-snapshot + if: ${{ matrix.version }} == "v2" + with: + snapshot: grpc_metrics.txt + artifact_key: grpc-metrics-${{ matrix.version }}.txt + - name: Upload coverage to codecov uses: ./.github/actions/upload-codecov with: diff --git a/.github/workflows/ci-e2e-kafka.yml b/.github/workflows/ci-e2e-kafka.yml index c6b3750b420..5f111c12285 100644 --- a/.github/workflows/ci-e2e-kafka.yml +++ b/.github/workflows/ci-e2e-kafka.yml @@ -28,6 +28,10 @@ jobs: - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Fetch git tags + run: | + git fetch --prune --unshallow --tags + - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: go-version: 1.23.x @@ -35,6 +39,12 @@ jobs: - name: Run kafka integration tests id: test-execution run: bash scripts/e2e/kafka.sh -j ${{ matrix.jaeger-version }} -v ${{ matrix.kafka-version }} + + - uses: ./.github/actions/verify-metrics-snapshot + if: ${{ matrix.jaeger-version }} == "v2" + with: + snapshot: kafka_metrics.txt + artifact_key: kafka-metrics-${{ matrix.jaeger-version }}.txt - name: Upload coverage to codecov uses: ./.github/actions/upload-codecov diff --git a/.github/workflows/ci-e2e-memory.yaml b/.github/workflows/ci-e2e-memory.yaml index c8dd9123c8e..3dd605881a7 100644 --- a/.github/workflows/ci-e2e-memory.yaml +++ b/.github/workflows/ci-e2e-memory.yaml @@ -22,6 +22,10 @@ jobs: - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + - name: Fetch git tags + run: | + git fetch --prune --unshallow --tags + - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: go-version: 1.23.x @@ -29,7 +33,12 @@ jobs: - name: Run Memory storage integration tests run: | STORAGE=memory_v2 make jaeger-v2-storage-integration-test - + + - uses: ./.github/actions/verify-metrics-snapshot + with: + snapshot: memory_metrics.txt + artifact_key: memory-metrics.txt + - name: Upload coverage to codecov uses: ./.github/actions/upload-codecov with: diff --git a/.github/workflows/ci-e2e-opensearch.yml b/.github/workflows/ci-e2e-opensearch.yml index c4d79a3c2ad..2ff79b920d1 100644 --- a/.github/workflows/ci-e2e-opensearch.yml +++ b/.github/workflows/ci-e2e-opensearch.yml @@ -52,6 +52,12 @@ jobs: id: test-execution run: bash scripts/e2e/elasticsearch.sh ${{ matrix.version.distribution }} ${{ matrix.version.major }} ${{ matrix.version.jaeger }} + - uses: ./.github/actions/verify-metrics-snapshot + if: ${{ matrix.version.jaeger }} == "v2" + with: + snapshot: opensearch_metrics.txt + artifact_key: opensearch-metrics-${{ matrix.version.major }}.txt + - name: Upload coverage to codecov uses: ./.github/actions/upload-codecov with: diff --git a/.gitignore b/.gitignore index e8719d8386d..95bc230a2d3 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,4 @@ sha256sum.combined.txt resource.syso .gocache test-results.json +.metrics/ \ No newline at end of file diff --git a/cmd/jaeger/internal/integration/e2e_integration.go b/cmd/jaeger/internal/integration/e2e_integration.go index 5882d1f42e0..59f566cb114 100644 --- a/cmd/jaeger/internal/integration/e2e_integration.go +++ b/cmd/jaeger/internal/integration/e2e_integration.go @@ -99,11 +99,32 @@ func (s *E2EStorageIntegration) e2eInitialize(t *testing.T, storage string) { require.NoError(t, err) t.Cleanup(func() { + scrapeMetrics(t, storage) require.NoError(t, s.TraceReader.(io.Closer).Close()) require.NoError(t, s.TraceWriter.(io.Closer).Close()) }) } +func scrapeMetrics(t *testing.T, storage string) { + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://localhost:8888/metrics", nil) + require.NoError(t, err) + + client := &http.Client{} + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + outputDir := "../../../../.metrics" + require.NoError(t, os.MkdirAll(outputDir, os.ModePerm)) + + metricsFile, err := os.Create(fmt.Sprintf("%s/%v_metrics.txt", outputDir, storage)) + require.NoError(t, err) + defer metricsFile.Close() + + _, err = io.Copy(metricsFile, resp.Body) + require.NoError(t, err) +} + func createStorageCleanerConfig(t *testing.T, configFile string, storage string) string { data, err := os.ReadFile(configFile) require.NoError(t, err) diff --git a/cmd/jaeger/internal/integration/trace_writer.go b/cmd/jaeger/internal/integration/trace_writer.go index c5286aa44d0..999ce34fb4d 100644 --- a/cmd/jaeger/internal/integration/trace_writer.go +++ b/cmd/jaeger/internal/integration/trace_writer.go @@ -81,7 +81,7 @@ func (w *traceWriter) WriteTraces(ctx context.Context, td ptrace.Traces) error { scope ptrace.ScopeSpans resource ptrace.ResourceSpans ) - + if spanCount == MaxChunkSize { err = w.exporter.ConsumeTraces(ctx, currentChunk) currentChunk = ptrace.NewTraces() diff --git a/scripts/e2e/compare_metrics.py b/scripts/e2e/compare_metrics.py new file mode 100644 index 00000000000..a8e94b52e4a --- /dev/null +++ b/scripts/e2e/compare_metrics.py @@ -0,0 +1,74 @@ +# Copyright (c) 2024 The Jaeger Authors. +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import sys +from difflib import unified_diff +from bisect import insort +from prometheus_client.parser import text_string_to_metric_families + +def read_metric_file(file_path): + with open(file_path, 'r') as f: + return f.readlines() + +def parse_metrics(content): + metrics = [] + for family in text_string_to_metric_families(content): + for sample in family.samples: + labels = dict(sample.labels) + #simply pop undesirable metric labels + labels.pop('service_instance_id',None) + label_pairs = sorted(labels.items(), key=lambda x: x[0]) + label_str = ','.join(f'{k}="{v}"' for k,v in label_pairs) + metric = f"{family.name}{{{label_str}}}" + insort(metrics , metric) + + return metrics + + +def generate_diff(file1_content, file2_content): + if isinstance(file1_content, list): + file1_content = ''.join(file1_content) + if isinstance(file2_content, list): + file2_content = ''.join(file2_content) + + metrics1 = parse_metrics(file1_content) + metrics2 = parse_metrics(file2_content) + + diff = unified_diff(metrics1, metrics2,lineterm='',n=0) + + return '\n'.join(diff) + +def write_diff_file(diff_lines, output_path): + + with open(output_path, 'w') as f: + f.write(diff_lines) + f.write('\n') # Add final newline + print(f"Diff file successfully written to: {output_path}") + +def main(): + parser = argparse.ArgumentParser(description='Generate diff between two Jaeger metric files') + parser.add_argument('--file1', help='Path to first metric file') + parser.add_argument('--file2', help='Path to second metric file') + parser.add_argument('--output', '-o', default='metrics_diff.txt', + help='Output diff file path (default: metrics_diff.txt)') + + args = parser.parse_args() + + # Read input files + file1_lines = read_metric_file(args.file1) + file2_lines = read_metric_file(args.file2) + + # Generate diff + diff_lines = generate_diff(file1_lines, file2_lines, str(args.file1), str(args.file2)) + + # Check if there are any differences + if not diff_lines: + print("No differences found between the metric files.") + sys.exit(0) + + # Write diff to output file + write_diff_file(diff_lines, args.output) + +if __name__ == '__main__': + main() \ No newline at end of file