add info about test failures

README.md

@@ -18,6 +18,16 @@ STATUS_TOKEN=<TOKEN>
 
 The token only needs read access to the repository.
 
+A lot of API calls are made to gather flaky tests statistics. If you would like to
+use a different API token for it, you can add it to the .env file:
+```
+FLAKY_TESTS_TOKEN=<ANOTHER_TOKEN>
+```
+
+If not provided, the `STATUS_TOKEN` will be used.
+
+**NOTE** in dev mode, gathering flaky tests is mocked.
+
 ## Running the application in dev mode
 
 You can run your application in dev mode that enables live coding using:

ddl.sql

@@ -0,0 +1,43 @@
+CREATE SEQUENCE hibernate_sequence
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER TABLE hibernate_sequence OWNER TO quarkus;
+
+SET default_tablespace = '';
+
+SET default_table_access_method = heap;
+
+CREATE TABLE testexecution (
+                                      id bigint NOT NULL,
+                                      successful boolean NOT NULL,
+                                      testname character varying(255),
+                                      job_id bigint
+);
+
+ALTER TABLE testexecution OWNER TO quarkus;
+
+CREATE TABLE testjob (
+                                id bigint NOT NULL,
+                                completedat timestamp without time zone,
+                                name character varying(255),
+                                sha character varying(255),
+                                url character varying(255)
+);
+
+ALTER TABLE testjob OWNER TO quarkus;
+
+ALTER TABLE ONLY testexecution
+    ADD CONSTRAINT testexecution_pkey PRIMARY KEY (id);
+
+ALTER TABLE ONLY testjob
+    ADD CONSTRAINT testjob_pkey PRIMARY KEY (id);
+
+ALTER TABLE ONLY testexecution
+    ADD CONSTRAINT fkptkmwmynles1g3xwdo7iudrun FOREIGN KEY (job_id) REFERENCES testjob(id);
+
+create index on testexecution (testname, id desc, successful);
+create index on testexecution using btree (testname);

pom.xml

@@ -64,6 +64,22 @@
             <artifactId>quarkus-prettytime</artifactId>
             <version>0.1.1</version>
         </dependency>
+
+        <!-- database: -->
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-jdbc-postgresql</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>io.quarkus</groupId>
+            <artifactId>quarkus-hibernate-orm-panache</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+        </dependency>
         <dependency>
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-junit5</artifactId>

src/main/java/io/quarkus/status/StatusResource.java

@@ -1,15 +1,20 @@
 package io.quarkus.status;
 
 import java.io.IOException;
+import java.util.List;
 
 import javax.inject.Inject;
+import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.MediaType;
 
 import io.quarkus.qute.CheckedTemplate;
 import io.quarkus.qute.TemplateInstance;
+import io.quarkus.status.flaky.TestExecution;
 import io.quarkus.status.model.Stats;
 import io.quarkus.status.model.Status;
 
@@ -29,6 +34,8 @@ public class StatusResource {
     public static class Templates {
         public static native TemplateInstance index(Status status);
         public static native TemplateInstance issues(Status status, Stats stats, boolean isBugs);
+        public static native TemplateInstance tests(Status status);
+        public static native TemplateInstance testResults(Status status, List<TestExecution> executions);
     }
 
     @GET
@@ -44,6 +51,24 @@ public TemplateInstance bugs() throws IOException {
         return Templates.issues(statusService.getStatus(), issuesService.getBugsMonthlyStats(), true);
     }
 
+    @GET
+    @Path("tests")
+    @Produces(MediaType.TEXT_HTML)
+    public TemplateInstance tests() throws IOException {
+        return Templates.tests(statusService.getStatus());
+    }
+
+    @GET
+    @Path("test-details/{testName}")
+    @Produces(MediaType.TEXT_HTML)
+    public TemplateInstance testResults(@PathParam("testName") String testName,
+                                        @QueryParam("page") @DefaultValue("0") Integer page,
+                                        @QueryParam("pageSize") @DefaultValue("40") Integer pageSize) throws IOException {
+        List<TestExecution> executions = TestExecution.find("testName = ?1 ORDER BY id DESC", testName)
+                .page(page, pageSize).list();
+        return Templates.testResults(statusService.getStatus(), executions);
+    }
+
     @GET
     @Path("enhancements")
     @Produces(MediaType.TEXT_HTML)

src/main/java/io/quarkus/status/flaky/TestDataInitializer.java

@@ -0,0 +1,105 @@
+package io.quarkus.status.flaky;
+
+import io.quarkus.arc.profile.UnlessBuildProfile;
+import io.quarkus.runtime.StartupEvent;
+
+import javax.enterprise.context.ApplicationScoped;
+import javax.enterprise.event.Observes;
+import javax.inject.Inject;
+import javax.transaction.Transactional;
+
+import java.util.Date;
+import java.util.Random;
+
+import static io.quarkus.status.flaky.TestStatisticsResource.RESULT_GROUP_SIZE;
+
+@ApplicationScoped
+@UnlessBuildProfile("prod")
+public class TestDataInitializer {
+    @Transactional
+    public void setUp(@Observes StartupEvent anything) {
+        TestJob testJob = new TestJob();
+        testJob.name = "JVM tests on Windows";
+        testJob.url = "https://example.com/test-result";
+        testJob.completedAt = new Date();
+        testJob.sha = "093245802938402934902341";
+        TestJob.persist(testJob);
+
+        for (int i = 0; i < 10; i++) {
+            TestExecution execution = new TestExecution();
+            execution.job = testJob;
+            execution.testName = "com.example.restclient.it.SomeTest.alwaysFailingTest";
+            execution.successful = false;
+            TestExecution.persist(execution);
+        }
+        for (int i = 0; i < 10; i++) {
+            TestExecution execution = new TestExecution();
+            execution.job = testJob;
+            execution.testName = "always.succeeding.test";
+            execution.successful = true;
+            TestExecution.persist(execution);
+        }
+
+
+        for (int i = 0; i < RESULT_GROUP_SIZE; i++) {
+            TestExecution execution = new TestExecution();
+            execution.job = testJob;
+            execution.testName = "half.failing.test";
+            execution.successful = (i % 2) == 0;
+            TestExecution.persist(execution);
+        }
+        for (int i = 0; i < RESULT_GROUP_SIZE * 2; i++) {
+            TestExecution execution = new TestExecution();
+            execution.job = testJob;
+            execution.testName = "test.failing.in.the.past";
+            execution.successful = i >= RESULT_GROUP_SIZE;
+            TestExecution.persist(execution);
+        }
+        for (int i = 0; i < RESULT_GROUP_SIZE * 2; i++) {
+            TestExecution execution = new TestExecution();
+            execution.job = testJob;
+            execution.testName = "test.successful.in.the.past";
+            execution.successful = i <= RESULT_GROUP_SIZE;
+            TestExecution.persist(execution);
+        }
+    }
+
+    @Inject
+    PerfTestInitializer perfTestInitializer;
+
+    public void createDataForPerfTest(@Observes StartupEvent anything) {
+        if (!Boolean.TRUE.toString().equalsIgnoreCase(System.getenv("generate-perf-test-data"))) {
+            return;
+        }
+        Random r = new Random();
+        for (int i = 0; i < 600; i++) {
+            String testName = "test.successful.in.the.past" + r.nextInt();
+            perfTestInitializer.initTestResults(testName);
+        }
+        System.out.println("\ndone");
+    }
+
+    @ApplicationScoped
+    public static class PerfTestInitializer {
+
+        public static final int TEST_ROWS_FOR_TEST = 50;
+
+        @Transactional
+        void initTestResults(String testName) {
+            TestJob testJob = new TestJob();
+            testJob.name = "JVM tests on Linux";
+            testJob.url = "https://example.com/test-result/2";
+            testJob.completedAt = new Date();
+            testJob.sha = "123132423412313242341";
+            TestJob.persist(testJob);
+            Random r = new Random();
+            for (int i = 0; i < TEST_ROWS_FOR_TEST; i++) {
+                TestExecution execution = new TestExecution();
+                execution.job = testJob;
+                execution.testName = testName;
+                execution.successful = r.nextDouble() > 0.1;
+                TestExecution.persist(execution);
+            }
+        }
+    }
+}

src/main/java/io/quarkus/status/flaky/TestExecution.java

@@ -0,0 +1,29 @@
+package io.quarkus.status.flaky;
+
+import io.quarkus.hibernate.orm.panache.PanacheEntity;
+
+import javax.persistence.Entity;
+import javax.persistence.Index;
+import javax.persistence.ManyToOne;
+import javax.persistence.Table;
+
+@Entity
+@Table(indexes = {
+        @Index(columnList = "testName"),
+        @Index(columnList = "testName, id desc, successful")
+})
+public class TestExecution extends PanacheEntity {
+    public String testName;
+    public boolean successful;
+    @ManyToOne
+    public TestJob job;
+
+    @Override
+    public String toString() {
+        return "TestExecution{" +
+                "testName='" + testName + '\'' +
+                ", successful=" + successful +
+                ", id=" + id +
+                '}';
+    }
+}

src/main/java/io/quarkus/status/flaky/TestJob.java

@@ -0,0 +1,14 @@
+package io.quarkus.status.flaky;
+
+import io.quarkus.hibernate.orm.panache.PanacheEntity;
+
+import javax.persistence.Entity;
+import java.util.Date;
+
+@Entity
+public class TestJob extends PanacheEntity {
+    public String url;
+    public String name;
+    public Date completedAt;
+    public String sha;
+}

src/main/java/io/quarkus/status/flaky/TestStatisticsResource.java

@@ -0,0 +1,82 @@
+package io.quarkus.status.flaky;
+
+import javax.enterprise.context.ApplicationScoped;
+import javax.persistence.EntityManager;
+import javax.persistence.PersistenceContext;
+import javax.persistence.Query;
+import javax.ws.rs.BadRequestException;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
+import java.util.stream.Collectors;
+
+@Path("/test-statistics")
+@ApplicationScoped
+public class TestStatisticsResource {
+    public static final int TESTS_TABLE_SIZE = 20;
+    public static final int RESULT_GROUP_SIZE = 40;
+
+    @PersistenceContext
+    EntityManager entityManager;
+
+    private final Map<String, List<TestStatistics>> queryCache = new WeakHashMap<>();
+
+    @GET
+    @Produces(MediaType.APPLICATION_JSON)
+    public List<TestStatistics> getStats(@QueryParam("testQuery") String testQueryParam) {
+        String testQuery = testQueryParam == null || testQueryParam.isBlank()
+                ? "%"
+                : '%' + testQueryParam + '%';
+        if (queryCache.containsKey(testQuery)) {
+            return queryCache.get(testQuery);
+        }
+
+        if (!testQuery.equals("%")) {
+            Query countQuery = entityManager.createNativeQuery(
+                    "select count(t.*) from (select distinct testname from testexecution where testname like :test) t");
+            countQuery.setParameter("test", testQuery);
+            BigInteger count = (BigInteger) countQuery.getSingleResult();
+
+            if (count.intValue() > 100) {
+                throw new BadRequestException(Response.status(400)
+                        .entity("Too many tests matching the query found, please use a more specific query").build());
+            }
+        }
+
+        Query query = entityManager.createNativeQuery("with partitioned_results as " +
+                "(select id, testname, successful\\:\\:int as success, row_number() over (partition by testname order by id desc) from testexecution)" +
+                "select testname, avg(success) as success_rate, sum(success), count(id) from partitioned_results " +
+                "where testname like :test and row_number <= " + RESULT_GROUP_SIZE + " group by testname order by success_rate asc limit " + TESTS_TABLE_SIZE);
+
+        query.setParameter("test", testQuery);
+        List<Object[]> stats = query.getResultList();
+        List<TestStatistics> result = stats.stream()
+                .map(TestStatistics::new)
+                .collect(Collectors.toList());
+
+        queryCache.put(testQuery, result);
+        return result;
+    }
+
+    public static class TestStatistics {
+        public String name;
+        public double failureRatio;
+        public int successCount;
+        public int executionCount;
+
+        public TestStatistics(Object[] result) {
+            this.name = (String) result[0];
+            this.failureRatio = 1 - ((BigDecimal) result[1]).doubleValue();
+            this.successCount = ((BigInteger) result[2]).intValue();
+            this.executionCount = ((BigInteger) result[3]).intValue();
+        }
+    }
+}