bechmark

nawaz1991 · Nov 13, 2023 · f7096e4 · f7096e4
1 parent cf49c4b
commit f7096e4
Show file tree

Hide file tree

Showing 6 changed files with 182 additions and 5 deletions.
diff --git a/.github/convert_json_to_html.py b/.github/convert_json_to_html.py
@@ -0,0 +1,111 @@
+import json
+
+# load json data from benchmark_results.json
+with open("benchmark_results.json", "r") as file:
+    data = json.load(file)
+
+
+# Function to generate an HTML table from a dictionary
+def generate_table_from_dict(data_dict, title=None):
+    html = f"<h2>{title}</h2>" if title else ""
+    html += "<table class='info-table'>"
+    for key, value in data_dict.items():
+        html += "<tr>"
+        html += f"<td class='key-cell'><strong>{key}</strong></td>"
+        if isinstance(value, list):
+            html += "<td><table class='inner-table'>"
+            for item in value:
+                html += "<tr>"
+                if isinstance(item, dict):
+                    for k, v in item.items():
+                        html += f"<td>{k}: {v}</td>"
+                else:
+                    html += f"<td>{item}</td>"
+                html += "</tr>"
+            html += "</table></td>"
+        else:
+            html += f"<td>{value}</td>"
+        html += "</tr>"
+    html += "</table>"
+    return html
+
+# Function to generate an HTML table for benchmarks
+def generate_benchmarks_table(benchmarks):
+    if not benchmarks:
+        return ""
+
+    # Headers to be removed
+    headers_to_remove = ['family_index', 'per_family_instance_index', 'run_name', 'run_type', 'repetitions', 'repetition_index', 'threads', 'time_unit']
+
+    html = "<h2>Benchmarks</h2>"
+    html += "<table class='benchmark-table'><tr>"
+    html += "<th>Index</th>"
+
+    # Add headers
+    for key in benchmarks[0].keys():
+        if key not in headers_to_remove:
+            new_key = key
+            if key == 'name':
+                new_key = 'Benchmark'
+            elif key == 'real_time':
+                new_key = 'Real Time (us)'
+            elif key == 'cpu_time':
+                new_key = 'CPU Time (us)'
+            html += f"<th>{new_key}</th>"
+
+    html += "</tr>"
+
+    # Add rows
+    for benchmark in benchmarks:
+        html += "<tr>"
+        # Add index
+        html += f"<td>{benchmark['family_index']+1}</td>"
+        for key, value in benchmark.items():
+            if key not in headers_to_remove:
+                if key == 'name':
+                    value = value.replace('OASValidatorPerf/', '').split('/min_time')[0]
+                elif key in ['real_time', 'cpu_time']:
+                    value = f"{value:.3f}"
+                html += f"<td>{value}</td>"
+        html += "</tr>"
+
+    html += "</table>"
+    return html
+
+# Main function to generate the complete HTML page
+def generate_html(data):
+    html = """
+    <html>
+    <head>
+        <title>Benchmark Report</title>
+        <style>
+            body { font-family: Arial, sans-serif; margin: 0; padding: 0; background-color: #f4f4f4; color: #333; }
+            .container { width: 80%; margin: 20px auto; }
+            h1 { text-align: center; }
+            .info-table, .benchmark-table { width: 100%; border-collapse: collapse; margin-bottom: 20px; }
+            .info-table td, .benchmark-table th, .benchmark-table td { border: 1px solid #ddd; padding: 8px; text-align: left; }
+            .info-table .key-cell { width: 20%; font-weight: bold; }
+            .inner-table { width: 100%; margin-top: 5px; }
+            .inner-table td { border: none; padding: 4px; font-size: 0.9em; }
+            .benchmark-table th { background-color: #4CAF50; color: white; }
+            tr:nth-child(even) { background-color: #f2f2f2; }
+        </style>
+    </head>
+    <body>
+        <div class="container">
+            <h1>Benchmark Report</h1>
+            """ + generate_table_from_dict(data["context"], "Context") + generate_benchmarks_table(data["benchmarks"]) + """
+        </div>
+    </body>
+    </html>
+    """
+    return html
+
+# Generate HTML page
+html_page = generate_html(data)
+
+# Save the HTML to a file
+with open("benchmark_report.html", "w") as file:
+    file.write(html_page)
+
+print("HTML page generated and saved as benchmark_report.html")
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -0,0 +1,51 @@
+name: Benchmark
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: 'recursive'
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc g++ cmake lcov gcovr 
+          sudo apt-get install -y libgtest-dev
+          sudo apt install -y libbenchmark-dev
+          cd /usr/src/gtest
+          sudo cmake CMakeLists.txt
+          sudo make
+          sudo cp ./lib/libgtest*.a /usr/lib          
+
+      - name: Configure CMake
+        run: cmake -S . -B build -DBUILD_PERF=ON
+
+      - name: Build
+        run: cmake --build build --config Release
+
+      - name: Run benchmark
+        run: |
+          cd build/test/perftest
+          ./cpp-oasvalidator-perftests > --benchmark_out=benchmark_result.json --benchmark_format=json > benchmark_result.json
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Generate HTML Page
+        run: python .github/workflows/convert_json_to_html.py
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./build/test/perftest/benchmark_report.html
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -7,7 +7,7 @@ on:
     branches: [ main ]
 
 jobs:
-  build-and-test:
+  build:
     runs-on: ubuntu-latest
 
     steps:

diff --git a/.github/workflows/code_cov.yml b/.github/workflows/code_cov.yml
@@ -7,7 +7,7 @@ on:
     branches: [ main ]
 
 jobs:
-  build-and-test:
+  code-coverage:
     runs-on: ubuntu-latest
 
     steps:

diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml
@@ -7,7 +7,7 @@ on:
     branches: [ main ]
 
 jobs:
-  build-and-test:
+  unittest:
     runs-on: ubuntu-latest
 
     steps:

diff --git a/README.md b/README.md
@@ -26,7 +26,8 @@ With support for OpenAPI 3.x, this library streamlines the process of validating
         2. [Building and Installing](#512-building-and-installing)
         3. [Running the Tests](#513-running-the-tests)
         4. [Generating Code Coverage Report](#514-generating-code-coverage-report)
-        5. [Running the Example](#515-running-the-example)
+        5. [Performance Benchmarking](#515-performance-benchmarking)
+        6. [Running the Example](#516-running-the-example)
     2. [Initialization](#52--initialization-)
 6. [Conclusion](#6-conclusion-)
 7. [License](#7-license-)
@@ -199,7 +200,21 @@ To generate the code coverage report, follow the steps below:
     ```
     The coverage report will be generated in the `build/covhtml-cpp-oasvalidator/` directory. Open `index.html` in your browser to view the report.
 
-#### 5.1.5 Running the Example
+### 5.1.5 Performance Benchmarking
+
+To run the performance benchmark, follow the steps below:
+1. Navigate to the root directory of the project.
+2. Create a new directory named `build` and navigate into it:
+   ```bash
+    mkdir build
+    cd build
+    cmake .. -DBUILD_PERF=ON
+    cd test/perftest/
+    make
+    ./cpp-oasvalidator-perftests
+    ```
+
+#### 5.1.6 Running the Example
 
 To run the example, follow the steps below:
 1. Navigate to the root directory of the project.
-Original file line number
+Diff line change
@@ Expand Up / @@ -7,7 +7,7 @@ on: @@
         branches: [ main ]
     jobs:
-      build-and-test:
+      build:
         runs-on: ubuntu-latest
         steps:
@@ Expand Down @@