From 58770c6f7203763df1f28170ebbce73f84ddd134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Ta=C3=AFeb?= Date: Fri, 27 Dec 2024 17:49:34 +0100 Subject: [PATCH 01/10] feat: api stats --- package.json | 1 + pnpm-lock.yaml | 8 +++ prisma/schema.prisma | 2 +- prisma/sql/northStarStatQuery.sql | 16 +++++ src/app/api/stats/route.ts | 71 ++++++++++++++++++++++ src/helpers/dateUtils.ts | 2 + src/lib/prisma/prisma-analytics-queries.ts | 11 ++++ 7 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 prisma/sql/northStarStatQuery.sql create mode 100644 src/app/api/stats/route.ts diff --git a/package.json b/package.json index 0ce79108..58472985 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "@sentry/nextjs": "7.120.1", "@splidejs/react-splide": "^0.7.12", "clsx": "^2.1.1", + "date-fns": "^4.1.0", "html-to-image": "^1.11.11", "jspdf": "^2.5.2", "leaflet": "^1.9.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 52b72570..a79eb194 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -44,6 +44,9 @@ importers: clsx: specifier: ^2.1.1 version: 2.1.1 + date-fns: + specifier: ^4.1.0 + version: 4.1.0 html-to-image: specifier: ^1.11.11 version: 1.11.11 @@ -1355,6 +1358,9 @@ packages: resolution: {integrity: sha512-t/Ygsytq+R995EJ5PZlD4Cu56sWa8InXySaViRzw9apusqsOO2bQP+SbYzAhR0pFKoB+43lYy8rWban9JSuXnA==} engines: {node: '>= 0.4'} + date-fns@4.1.0: + resolution: {integrity: sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==} + debug@3.2.7: resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -4669,6 +4675,8 @@ snapshots: es-errors: 1.3.0 is-data-view: 1.0.1 + date-fns@4.1.0: {} + debug@3.2.7: dependencies: ms: 2.1.3 diff --git a/prisma/schema.prisma b/prisma/schema.prisma index facc1fe0..0f01bc0a 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,6 +1,6 @@ generator client { provider = "prisma-client-js" - previewFeatures = ["fullTextSearch"] + previewFeatures = ["fullTextSearch", "typedSql"] } datasource db { diff --git a/prisma/sql/northStarStatQuery.sql b/prisma/sql/northStarStatQuery.sql new file mode 100644 index 00000000..41a9e869 --- /dev/null +++ b/prisma/sql/northStarStatQuery.sql @@ -0,0 +1,16 @@ +-- @param {DateTime} $1:dateFrom Start date to use for computing stats +-- @param {String} $2:range Date range to use (year/month/week/day) +with score_table as (select date_trunc($2, created_at) as date1, + sum(CASE WHEN event_type = 'UPDATE_PROJET_SET_VISIBLE' THEN 1 ELSE -1 END) as score + from pfmv."Analytics" + WHERE event_type in ('UPDATE_PROJET_SET_VISIBLE', 'UPDATE_PROJET_SET_INVISIBLE') + and created_at >= $1 + group by 1 + order by 1 desc), + all_intervals as (SELECT date_trunc($2, ts) as date1 + FROM generate_series($1, now(), CONCAT('1 ',$2)::interval) AS ts + group by 1) +select all_intervals.date1::timestamp::date as "periode", coalesce(score, 0) as "score" +from score_table + right outer join all_intervals on all_intervals.date1 = score_table.date1 +order by 1; \ No newline at end of file diff --git a/src/app/api/stats/route.ts b/src/app/api/stats/route.ts new file mode 100644 index 00000000..394b13cc --- /dev/null +++ b/src/app/api/stats/route.ts @@ -0,0 +1,71 @@ +import { NextRequest, NextResponse } from "next/server"; +import z from "zod"; +import { add } from "date-fns/add"; +import { startOfYear } from "date-fns/startOfYear"; +import { startOfMonth } from "date-fns/startOfMonth"; +import { startOfWeek } from "date-fns/startOfWeek"; +import { startOfDay } from "date-fns/startOfDay"; +import { getNorthStarStats } from "@/src/lib/prisma/prisma-analytics-queries"; +import { fr } from "date-fns/locale/fr"; + +const StatsRouteSchema = z.object({ + since: z + .number() + .positive() + .max(5000, { message: "Veuillez rentrer une valeur inférieure à 5000 pour le paramètre since" }), + periodicity: z.enum(["year", "month", "week", "day"]), +}); + +interface StatOutputRecord { + value: number; + date: Date; +} + +type StatOuput = { + description?: string; + stats: StatOutputRecord[]; +}; + +export async function POST(request: NextRequest) { + const requestBody = await request.json(); + const parsedRequest = StatsRouteSchema.safeParse(requestBody); + if (!parsedRequest.success) { + const { errors } = parsedRequest.error; + return NextResponse.json({ error: { message: "Invalid request", errors } }, { status: 400 }); + } else { + const { since: nbIntervals, periodicity } = parsedRequest.data; + let dateBeginOfLastPeriod = new Date(); + switch (periodicity) { + case "year": + dateBeginOfLastPeriod = startOfYear(new Date()); + break; + case "month": + dateBeginOfLastPeriod = startOfMonth(new Date()); + break; + case "week": + dateBeginOfLastPeriod = startOfWeek(new Date(), { weekStartsOn: 1, locale: fr }); + break; + case "day": + dateBeginOfLastPeriod = startOfDay(new Date()); + break; + } + dateBeginOfLastPeriod = add(dateBeginOfLastPeriod, { + minutes: -dateBeginOfLastPeriod.getTimezoneOffset(), + }); + + const dateBeginOfFirstPeriod = add(dateBeginOfLastPeriod, { + ...(periodicity === "year" && { years: 1 - nbIntervals }), + ...(periodicity === "month" && { months: 1 - nbIntervals }), + ...(periodicity === "week" && { weeks: 1 - nbIntervals }), + ...(periodicity === "day" && { days: 1 - nbIntervals }), + }); + const results = await getNorthStarStats({ dateFrom: dateBeginOfFirstPeriod, range: periodicity }); + const responseEndpoint: StatOuput = { + stats: results.map((result) => ({ + value: Number(result.score)!, + date: result.periode!, + })), + }; + return NextResponse.json(responseEndpoint); + } +} diff --git a/src/helpers/dateUtils.ts b/src/helpers/dateUtils.ts index 776d5e07..bdcd3a57 100644 --- a/src/helpers/dateUtils.ts +++ b/src/helpers/dateUtils.ts @@ -1,3 +1,5 @@ +export type DateRange = "day" | "week" | "month" | "year"; + export const FAR_FUTURE = new Date(3024, 0, 0, 1); export const removeDaysToDate = (date: Date, nbDays: number) => new Date(date.getTime() - nbDays * 24 * 60 * 60 * 1000); diff --git a/src/lib/prisma/prisma-analytics-queries.ts b/src/lib/prisma/prisma-analytics-queries.ts index ef74c5ac..ef93429e 100644 --- a/src/lib/prisma/prisma-analytics-queries.ts +++ b/src/lib/prisma/prisma-analytics-queries.ts @@ -1,5 +1,7 @@ import { Analytics } from "@prisma/client"; import { prismaClient } from "./prismaClient"; +import { northStarStatQuery } from "@prisma/client/sql"; +import { DateRange } from "@/src/helpers/dateUtils"; type AnalyticsProps = Omit; @@ -14,3 +16,12 @@ export const createAnalytic = async (analytics: AnalyticsProps): Promise { + return prismaClient.$queryRawTyped(northStarStatQuery(params.dateFrom, params.range)); +}; From 698b1c14a731779a67ae374f555efa3c889d4549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Ta=C3=AFeb?= Date: Fri, 27 Dec 2024 18:00:13 +0100 Subject: [PATCH 02/10] feat: api stats should be GET and not POST --- src/app/api/stats/route.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/app/api/stats/route.ts b/src/app/api/stats/route.ts index 394b13cc..2690b99d 100644 --- a/src/app/api/stats/route.ts +++ b/src/app/api/stats/route.ts @@ -26,9 +26,11 @@ type StatOuput = { stats: StatOutputRecord[]; }; -export async function POST(request: NextRequest) { - const requestBody = await request.json(); - const parsedRequest = StatsRouteSchema.safeParse(requestBody); +export async function GET(request: NextRequest) { + const parsedRequest = StatsRouteSchema.safeParse({ + since: +(request.nextUrl.searchParams.get("since") ?? 0), + periodicity: request.nextUrl.searchParams.get("periodicity"), + }); if (!parsedRequest.success) { const { errors } = parsedRequest.error; return NextResponse.json({ error: { message: "Invalid request", errors } }, { status: 400 }); From 0e8d6e00f68c733c77dfc1f82677dfcfada994cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Ta=C3=AFeb?= Date: Fri, 27 Dec 2024 18:12:40 +0100 Subject: [PATCH 03/10] fix: fix build --- src/lib/prisma/prisma-analytics-queries.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/prisma/prisma-analytics-queries.ts b/src/lib/prisma/prisma-analytics-queries.ts index ef93429e..9234c836 100644 --- a/src/lib/prisma/prisma-analytics-queries.ts +++ b/src/lib/prisma/prisma-analytics-queries.ts @@ -21,7 +21,11 @@ type GetNorthStarStatsProps = { dateFrom: Date; range: DateRange; }; +type NorthStarQueryRecord = { + periode: Date | null; + score: bigint | null; +}; -export const getNorthStarStats = async (params: GetNorthStarStatsProps) => { +export const getNorthStarStats = async (params: GetNorthStarStatsProps): Promise => { return prismaClient.$queryRawTyped(northStarStatQuery(params.dateFrom, params.range)); }; From 07245f1b3fa817eacb3d4c1b98af4621c8075f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Ta=C3=AFeb?= Date: Mon, 30 Dec 2024 08:54:31 +0100 Subject: [PATCH 04/10] fix: try to fix build --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 58472985..177cef66 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "scripts": { "predev:start": "only-include-used-icons", "predev:localAsProd": "only-include-used-icons", - "prebuild": "only-include-used-icons && npx prisma generate", + "prebuild": "only-include-used-icons && npx prisma generate --sql", "dev:db:migrate": "prisma migrate dev", "dev:db:push": "prisma db push", "dev:db:generateClient": "prisma generate", From 1e132ada4bf4b449142ff09c7d4a2aa2ac71b3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Ta=C3=AFeb?= Date: Mon, 6 Jan 2025 11:16:53 +0100 Subject: [PATCH 05/10] feat: use old prisma raw query way as new one does not work on CI CD. --- package.json | 2 +- prisma/schema.prisma | 2 +- prisma/sql/northStarStatQuery.sql | 16 -------------- src/lib/prisma/prisma-analytics-queries.ts | 25 +++++++++++++++++++--- 4 files changed, 24 insertions(+), 21 deletions(-) delete mode 100644 prisma/sql/northStarStatQuery.sql diff --git a/package.json b/package.json index 177cef66..58472985 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "scripts": { "predev:start": "only-include-used-icons", "predev:localAsProd": "only-include-used-icons", - "prebuild": "only-include-used-icons && npx prisma generate --sql", + "prebuild": "only-include-used-icons && npx prisma generate", "dev:db:migrate": "prisma migrate dev", "dev:db:push": "prisma db push", "dev:db:generateClient": "prisma generate", diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 0f01bc0a..facc1fe0 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,6 +1,6 @@ generator client { provider = "prisma-client-js" - previewFeatures = ["fullTextSearch", "typedSql"] + previewFeatures = ["fullTextSearch"] } datasource db { diff --git a/prisma/sql/northStarStatQuery.sql b/prisma/sql/northStarStatQuery.sql deleted file mode 100644 index 41a9e869..00000000 --- a/prisma/sql/northStarStatQuery.sql +++ /dev/null @@ -1,16 +0,0 @@ --- @param {DateTime} $1:dateFrom Start date to use for computing stats --- @param {String} $2:range Date range to use (year/month/week/day) -with score_table as (select date_trunc($2, created_at) as date1, - sum(CASE WHEN event_type = 'UPDATE_PROJET_SET_VISIBLE' THEN 1 ELSE -1 END) as score - from pfmv."Analytics" - WHERE event_type in ('UPDATE_PROJET_SET_VISIBLE', 'UPDATE_PROJET_SET_INVISIBLE') - and created_at >= $1 - group by 1 - order by 1 desc), - all_intervals as (SELECT date_trunc($2, ts) as date1 - FROM generate_series($1, now(), CONCAT('1 ',$2)::interval) AS ts - group by 1) -select all_intervals.date1::timestamp::date as "periode", coalesce(score, 0) as "score" -from score_table - right outer join all_intervals on all_intervals.date1 = score_table.date1 -order by 1; \ No newline at end of file diff --git a/src/lib/prisma/prisma-analytics-queries.ts b/src/lib/prisma/prisma-analytics-queries.ts index 9234c836..74745487 100644 --- a/src/lib/prisma/prisma-analytics-queries.ts +++ b/src/lib/prisma/prisma-analytics-queries.ts @@ -1,6 +1,7 @@ -import { Analytics } from "@prisma/client"; +/* eslint-disable max-len */ + +import { Analytics, Prisma } from "@prisma/client"; import { prismaClient } from "./prismaClient"; -import { northStarStatQuery } from "@prisma/client/sql"; import { DateRange } from "@/src/helpers/dateUtils"; type AnalyticsProps = Omit; @@ -27,5 +28,23 @@ type NorthStarQueryRecord = { }; export const getNorthStarStats = async (params: GetNorthStarStatsProps): Promise => { - return prismaClient.$queryRawTyped(northStarStatQuery(params.dateFrom, params.range)); + return prismaClient.$queryRaw( + Prisma.sql`with score_table as (select date_trunc(${params.range}, created_at) as date1, + sum(CASE WHEN event_type = 'UPDATE_PROJET_SET_VISIBLE' THEN 1 ELSE -1 END) as score + from pfmv."Analytics" + WHERE event_type in + ('UPDATE_PROJET_SET_VISIBLE', + 'UPDATE_PROJET_SET_INVISIBLE') + and created_at >= ${params.dateFrom} + group by 1 + order by 1 desc), + all_intervals as (SELECT date_trunc(${params.range}, ts) as date1 + FROM generate_series(${params.dateFrom}, now(), CONCAT('1 ', ${params.range})::interval) AS ts + group by 1) + select all_intervals.date1::timestamp::date as "periode", + coalesce(score, 0) as "score" + from score_table + right outer join all_intervals on all_intervals.date1 = score_table.date1 + order by 1;`, + ); }; From 2bcbc7f28dd7b4b17e7e1158576b41475e2319f5 Mon Sep 17 00:00:00 2001 From: mehdilouraoui Date: Tue, 7 Jan 2025 15:58:26 +0100 Subject: [PATCH 06/10] feat: change north star value --- src/app/api/stats/route.ts | 80 +++++++++++++++++++--- src/lib/prisma/prisma-analytics-queries.ts | 63 ++++++++++------- 2 files changed, 109 insertions(+), 34 deletions(-) diff --git a/src/app/api/stats/route.ts b/src/app/api/stats/route.ts index 2690b99d..2d7e6f81 100644 --- a/src/app/api/stats/route.ts +++ b/src/app/api/stats/route.ts @@ -7,6 +7,7 @@ import { startOfWeek } from "date-fns/startOfWeek"; import { startOfDay } from "date-fns/startOfDay"; import { getNorthStarStats } from "@/src/lib/prisma/prisma-analytics-queries"; import { fr } from "date-fns/locale/fr"; +import { DateRange } from "@/src/helpers/dateUtils"; const StatsRouteSchema = z.object({ since: z @@ -21,11 +22,16 @@ interface StatOutputRecord { date: Date; } -type StatOuput = { +type StatOutput = { description?: string; stats: StatOutputRecord[]; }; +type GetNorthStarStatsProps = { + dateFrom: Date; + range: DateRange; +}; + export async function GET(request: NextRequest) { const parsedRequest = StatsRouteSchema.safeParse({ since: +(request.nextUrl.searchParams.get("since") ?? 0), @@ -61,13 +67,69 @@ export async function GET(request: NextRequest) { ...(periodicity === "week" && { weeks: 1 - nbIntervals }), ...(periodicity === "day" && { days: 1 - nbIntervals }), }); - const results = await getNorthStarStats({ dateFrom: dateBeginOfFirstPeriod, range: periodicity }); - const responseEndpoint: StatOuput = { - stats: results.map((result) => ({ - value: Number(result.score)!, - date: result.periode!, - })), - }; - return NextResponse.json(responseEndpoint); + const results = await getNorthStarStats(dateBeginOfFirstPeriod); + const computedStats = computeStats({ dateFrom: dateBeginOfFirstPeriod, range: periodicity }, results); + + return NextResponse.json(computedStats); } } + +const computeStats = ( + params: GetNorthStarStatsProps, + projets: { + created_at: Date; + }[], +): StatOutput => { + const statsMap = new Map(); + const now = new Date(); + let dateFrom = new Date(params.dateFrom); + + while (dateFrom <= now) { + statsMap.set(dateFrom.toISOString(), 0); + switch (params.range) { + case "day": + dateFrom = add(dateFrom, { days: 1 }); + break; + case "week": + dateFrom = add(dateFrom, { weeks: 1 }); + break; + case "month": + dateFrom = add(dateFrom, { months: 1 }); + break; + case "year": + dateFrom = add(dateFrom, { years: 1 }); + break; + } + } + + projets.forEach((projet) => { + let periodeDate: Date; + switch (params.range) { + case "day": + periodeDate = startOfDay(projet.created_at); + break; + case "week": + periodeDate = startOfWeek(projet.created_at, { weekStartsOn: 1, locale: fr }); + break; + case "month": + periodeDate = startOfMonth(projet.created_at); + break; + case "year": + periodeDate = startOfYear(projet.created_at); + break; + } + const periodeKey = periodeDate.toISOString(); + statsMap.set(periodeKey, (statsMap.get(periodeKey) || 0) + 1); + }); + + const stats = Array.from(statsMap.entries()) + .map(([periode, count]) => ({ + date: new Date(periode), + value: Number(count), + })) + .sort((a, b) => a.date.getTime() - b.date.getTime()); + + return { + stats, + }; +}; diff --git a/src/lib/prisma/prisma-analytics-queries.ts b/src/lib/prisma/prisma-analytics-queries.ts index 74745487..2b624164 100644 --- a/src/lib/prisma/prisma-analytics-queries.ts +++ b/src/lib/prisma/prisma-analytics-queries.ts @@ -1,6 +1,6 @@ /* eslint-disable max-len */ -import { Analytics, Prisma } from "@prisma/client"; +import { Analytics } from "@prisma/client"; import { prismaClient } from "./prismaClient"; import { DateRange } from "@/src/helpers/dateUtils"; @@ -22,29 +22,42 @@ type GetNorthStarStatsProps = { dateFrom: Date; range: DateRange; }; -type NorthStarQueryRecord = { - periode: Date | null; - score: bigint | null; -}; -export const getNorthStarStats = async (params: GetNorthStarStatsProps): Promise => { - return prismaClient.$queryRaw( - Prisma.sql`with score_table as (select date_trunc(${params.range}, created_at) as date1, - sum(CASE WHEN event_type = 'UPDATE_PROJET_SET_VISIBLE' THEN 1 ELSE -1 END) as score - from pfmv."Analytics" - WHERE event_type in - ('UPDATE_PROJET_SET_VISIBLE', - 'UPDATE_PROJET_SET_INVISIBLE') - and created_at >= ${params.dateFrom} - group by 1 - order by 1 desc), - all_intervals as (SELECT date_trunc(${params.range}, ts) as date1 - FROM generate_series(${params.dateFrom}, now(), CONCAT('1 ', ${params.range})::interval) AS ts - group by 1) - select all_intervals.date1::timestamp::date as "periode", - coalesce(score, 0) as "score" - from score_table - right outer join all_intervals on all_intervals.date1 = score_table.date1 - order by 1;`, - ); +export const getNorthStarStats = async ( + dateFrom: GetNorthStarStatsProps["dateFrom"], +): Promise< + { + created_at: Date; + }[] +> => { + const projets = await prismaClient.projet.findMany({ + where: { + deleted_at: null, + created_at: { + gte: dateFrom, + }, + creator: { + NOT: [ + { + email: { + contains: "@ademe.fr", + }, + }, + { + email: { + contains: "@beta.gouv.fr", + }, + }, + { + email: "marie.racine@dihal.gouv.fr", + }, + ], + }, + }, + select: { + created_at: true, + }, + }); + + return projets; }; From b388476edddfc4162993d17380f576ce4b8630ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Ta=C3=AFeb?= Date: Mon, 13 Jan 2025 10:11:19 +0100 Subject: [PATCH 07/10] chore: mutualize code --- src/app/api/stats/route.ts | 43 ++++++-------------------------------- src/helpers/dateUtils.ts | 19 +++++++++++++++++ 2 files changed, 25 insertions(+), 37 deletions(-) diff --git a/src/app/api/stats/route.ts b/src/app/api/stats/route.ts index 2d7e6f81..b7d927e8 100644 --- a/src/app/api/stats/route.ts +++ b/src/app/api/stats/route.ts @@ -1,13 +1,8 @@ import { NextRequest, NextResponse } from "next/server"; import z from "zod"; import { add } from "date-fns/add"; -import { startOfYear } from "date-fns/startOfYear"; -import { startOfMonth } from "date-fns/startOfMonth"; -import { startOfWeek } from "date-fns/startOfWeek"; -import { startOfDay } from "date-fns/startOfDay"; import { getNorthStarStats } from "@/src/lib/prisma/prisma-analytics-queries"; -import { fr } from "date-fns/locale/fr"; -import { DateRange } from "@/src/helpers/dateUtils"; +import { DateRange, getBeginningDateOfRange } from "@/src/helpers/dateUtils"; const StatsRouteSchema = z.object({ since: z @@ -42,26 +37,13 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: { message: "Invalid request", errors } }, { status: 400 }); } else { const { since: nbIntervals, periodicity } = parsedRequest.data; - let dateBeginOfLastPeriod = new Date(); - switch (periodicity) { - case "year": - dateBeginOfLastPeriod = startOfYear(new Date()); - break; - case "month": - dateBeginOfLastPeriod = startOfMonth(new Date()); - break; - case "week": - dateBeginOfLastPeriod = startOfWeek(new Date(), { weekStartsOn: 1, locale: fr }); - break; - case "day": - dateBeginOfLastPeriod = startOfDay(new Date()); - break; - } - dateBeginOfLastPeriod = add(dateBeginOfLastPeriod, { + const dateBeginOfLastPeriod = getBeginningDateOfRange(new Date(), periodicity); + + const dateBeginOfLastPeriodWithOffset = add(dateBeginOfLastPeriod, { minutes: -dateBeginOfLastPeriod.getTimezoneOffset(), }); - const dateBeginOfFirstPeriod = add(dateBeginOfLastPeriod, { + const dateBeginOfFirstPeriod = add(dateBeginOfLastPeriodWithOffset, { ...(periodicity === "year" && { years: 1 - nbIntervals }), ...(periodicity === "month" && { months: 1 - nbIntervals }), ...(periodicity === "week" && { weeks: 1 - nbIntervals }), @@ -104,20 +86,7 @@ const computeStats = ( projets.forEach((projet) => { let periodeDate: Date; - switch (params.range) { - case "day": - periodeDate = startOfDay(projet.created_at); - break; - case "week": - periodeDate = startOfWeek(projet.created_at, { weekStartsOn: 1, locale: fr }); - break; - case "month": - periodeDate = startOfMonth(projet.created_at); - break; - case "year": - periodeDate = startOfYear(projet.created_at); - break; - } + periodeDate = getBeginningDateOfRange(projet.created_at, params.range); const periodeKey = periodeDate.toISOString(); statsMap.set(periodeKey, (statsMap.get(periodeKey) || 0) + 1); }); diff --git a/src/helpers/dateUtils.ts b/src/helpers/dateUtils.ts index bdcd3a57..a7d3356c 100644 --- a/src/helpers/dateUtils.ts +++ b/src/helpers/dateUtils.ts @@ -1,3 +1,9 @@ +import { startOfDay } from "date-fns/startOfDay"; +import { startOfWeek } from "date-fns/startOfWeek"; +import { fr } from "date-fns/locale/fr"; +import { startOfMonth } from "date-fns/startOfMonth"; +import { startOfYear } from "date-fns/startOfYear"; + export type DateRange = "day" | "week" | "month" | "year"; export const FAR_FUTURE = new Date(3024, 0, 0, 1); @@ -50,3 +56,16 @@ export const daysUntilDate = (targetDate: Date | null): number | null => { return Math.ceil((targetDate.getTime() - new Date().getTime()) / MS_PER_DAY); }; + +export const getBeginningDateOfRange = (dateValue: Date, range: DateRange): Date => { + switch (range) { + case "day": + return startOfDay(dateValue); + case "week": + return startOfWeek(dateValue, { weekStartsOn: 1, locale: fr }); + case "month": + return startOfMonth(dateValue); + case "year": + return startOfYear(dateValue); + } +}; From e26d27adcd5fde827ad6bd8e6efe20109b33eb63 Mon Sep 17 00:00:00 2001 From: mehdilouraoui Date: Mon, 13 Jan 2025 18:08:12 +0100 Subject: [PATCH 08/10] Revert "chore: mutualize code" This reverts commit b388476edddfc4162993d17380f576ce4b8630ce. --- src/app/api/stats/route.ts | 43 ++++++++++++++++++++++++++++++++------ src/helpers/dateUtils.ts | 19 ----------------- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/app/api/stats/route.ts b/src/app/api/stats/route.ts index b7d927e8..2d7e6f81 100644 --- a/src/app/api/stats/route.ts +++ b/src/app/api/stats/route.ts @@ -1,8 +1,13 @@ import { NextRequest, NextResponse } from "next/server"; import z from "zod"; import { add } from "date-fns/add"; +import { startOfYear } from "date-fns/startOfYear"; +import { startOfMonth } from "date-fns/startOfMonth"; +import { startOfWeek } from "date-fns/startOfWeek"; +import { startOfDay } from "date-fns/startOfDay"; import { getNorthStarStats } from "@/src/lib/prisma/prisma-analytics-queries"; -import { DateRange, getBeginningDateOfRange } from "@/src/helpers/dateUtils"; +import { fr } from "date-fns/locale/fr"; +import { DateRange } from "@/src/helpers/dateUtils"; const StatsRouteSchema = z.object({ since: z @@ -37,13 +42,26 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: { message: "Invalid request", errors } }, { status: 400 }); } else { const { since: nbIntervals, periodicity } = parsedRequest.data; - const dateBeginOfLastPeriod = getBeginningDateOfRange(new Date(), periodicity); - - const dateBeginOfLastPeriodWithOffset = add(dateBeginOfLastPeriod, { + let dateBeginOfLastPeriod = new Date(); + switch (periodicity) { + case "year": + dateBeginOfLastPeriod = startOfYear(new Date()); + break; + case "month": + dateBeginOfLastPeriod = startOfMonth(new Date()); + break; + case "week": + dateBeginOfLastPeriod = startOfWeek(new Date(), { weekStartsOn: 1, locale: fr }); + break; + case "day": + dateBeginOfLastPeriod = startOfDay(new Date()); + break; + } + dateBeginOfLastPeriod = add(dateBeginOfLastPeriod, { minutes: -dateBeginOfLastPeriod.getTimezoneOffset(), }); - const dateBeginOfFirstPeriod = add(dateBeginOfLastPeriodWithOffset, { + const dateBeginOfFirstPeriod = add(dateBeginOfLastPeriod, { ...(periodicity === "year" && { years: 1 - nbIntervals }), ...(periodicity === "month" && { months: 1 - nbIntervals }), ...(periodicity === "week" && { weeks: 1 - nbIntervals }), @@ -86,7 +104,20 @@ const computeStats = ( projets.forEach((projet) => { let periodeDate: Date; - periodeDate = getBeginningDateOfRange(projet.created_at, params.range); + switch (params.range) { + case "day": + periodeDate = startOfDay(projet.created_at); + break; + case "week": + periodeDate = startOfWeek(projet.created_at, { weekStartsOn: 1, locale: fr }); + break; + case "month": + periodeDate = startOfMonth(projet.created_at); + break; + case "year": + periodeDate = startOfYear(projet.created_at); + break; + } const periodeKey = periodeDate.toISOString(); statsMap.set(periodeKey, (statsMap.get(periodeKey) || 0) + 1); }); diff --git a/src/helpers/dateUtils.ts b/src/helpers/dateUtils.ts index a7d3356c..bdcd3a57 100644 --- a/src/helpers/dateUtils.ts +++ b/src/helpers/dateUtils.ts @@ -1,9 +1,3 @@ -import { startOfDay } from "date-fns/startOfDay"; -import { startOfWeek } from "date-fns/startOfWeek"; -import { fr } from "date-fns/locale/fr"; -import { startOfMonth } from "date-fns/startOfMonth"; -import { startOfYear } from "date-fns/startOfYear"; - export type DateRange = "day" | "week" | "month" | "year"; export const FAR_FUTURE = new Date(3024, 0, 0, 1); @@ -56,16 +50,3 @@ export const daysUntilDate = (targetDate: Date | null): number | null => { return Math.ceil((targetDate.getTime() - new Date().getTime()) / MS_PER_DAY); }; - -export const getBeginningDateOfRange = (dateValue: Date, range: DateRange): Date => { - switch (range) { - case "day": - return startOfDay(dateValue); - case "week": - return startOfWeek(dateValue, { weekStartsOn: 1, locale: fr }); - case "month": - return startOfMonth(dateValue); - case "year": - return startOfYear(dateValue); - } -}; From a3890aff49ce3ebae3fd8df4767479cc0efa22c1 Mon Sep 17 00:00:00 2001 From: mehdilouraoui Date: Mon, 13 Jan 2025 19:27:49 +0100 Subject: [PATCH 09/10] back to raw query --- prisma/schema.prisma | 2 +- prisma/sql/northStartStatQuery.sql | 16 ++++++++++++++++ src/lib/prisma/prisma-analytics-queries.ts | 5 +++++ src/types/sql.d.ts | 3 +++ 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 prisma/sql/northStartStatQuery.sql create mode 100644 src/types/sql.d.ts diff --git a/prisma/schema.prisma b/prisma/schema.prisma index facc1fe0..0f01bc0a 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,6 +1,6 @@ generator client { provider = "prisma-client-js" - previewFeatures = ["fullTextSearch"] + previewFeatures = ["fullTextSearch", "typedSql"] } datasource db { diff --git a/prisma/sql/northStartStatQuery.sql b/prisma/sql/northStartStatQuery.sql new file mode 100644 index 00000000..41a9e869 --- /dev/null +++ b/prisma/sql/northStartStatQuery.sql @@ -0,0 +1,16 @@ +-- @param {DateTime} $1:dateFrom Start date to use for computing stats +-- @param {String} $2:range Date range to use (year/month/week/day) +with score_table as (select date_trunc($2, created_at) as date1, + sum(CASE WHEN event_type = 'UPDATE_PROJET_SET_VISIBLE' THEN 1 ELSE -1 END) as score + from pfmv."Analytics" + WHERE event_type in ('UPDATE_PROJET_SET_VISIBLE', 'UPDATE_PROJET_SET_INVISIBLE') + and created_at >= $1 + group by 1 + order by 1 desc), + all_intervals as (SELECT date_trunc($2, ts) as date1 + FROM generate_series($1, now(), CONCAT('1 ',$2)::interval) AS ts + group by 1) +select all_intervals.date1::timestamp::date as "periode", coalesce(score, 0) as "score" +from score_table + right outer join all_intervals on all_intervals.date1 = score_table.date1 +order by 1; \ No newline at end of file diff --git a/src/lib/prisma/prisma-analytics-queries.ts b/src/lib/prisma/prisma-analytics-queries.ts index 2b624164..a1705d8d 100644 --- a/src/lib/prisma/prisma-analytics-queries.ts +++ b/src/lib/prisma/prisma-analytics-queries.ts @@ -3,6 +3,7 @@ import { Analytics } from "@prisma/client"; import { prismaClient } from "./prismaClient"; import { DateRange } from "@/src/helpers/dateUtils"; +import { northStartStatQuery } from "@prisma/sql"; type AnalyticsProps = Omit; @@ -23,6 +24,10 @@ type GetNorthStarStatsProps = { range: DateRange; }; +export const getNorthStarStatsRaw = async (params: GetNorthStarStatsProps) => { + return prismaClient.$queryRawTyped(northStartStatQuery(params.dateFrom, params.range)); +}; + export const getNorthStarStats = async ( dateFrom: GetNorthStarStatsProps["dateFrom"], ): Promise< diff --git a/src/types/sql.d.ts b/src/types/sql.d.ts new file mode 100644 index 00000000..755c736d --- /dev/null +++ b/src/types/sql.d.ts @@ -0,0 +1,3 @@ +declare module "@prisma/sql" { + export function northStartStatQuery(dateFrom: Date, range: string): string; +} From b60cafee92d102fed26d6c2ec5bbebcb87c96c9e Mon Sep 17 00:00:00 2001 From: mehdilouraoui Date: Tue, 14 Jan 2025 10:41:52 +0100 Subject: [PATCH 10/10] feat: use raw query --- prisma/schema.prisma | 2 +- prisma/sql/northStartStatQuery.sql | 16 ---- src/app/api/stats/route.ts | 100 ++++----------------- src/lib/prisma/prisma-analytics-queries.ts | 71 ++++++--------- src/types/sql.d.ts | 3 - 5 files changed, 48 insertions(+), 144 deletions(-) delete mode 100644 prisma/sql/northStartStatQuery.sql delete mode 100644 src/types/sql.d.ts diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 0f01bc0a..facc1fe0 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -1,6 +1,6 @@ generator client { provider = "prisma-client-js" - previewFeatures = ["fullTextSearch", "typedSql"] + previewFeatures = ["fullTextSearch"] } datasource db { diff --git a/prisma/sql/northStartStatQuery.sql b/prisma/sql/northStartStatQuery.sql deleted file mode 100644 index 41a9e869..00000000 --- a/prisma/sql/northStartStatQuery.sql +++ /dev/null @@ -1,16 +0,0 @@ --- @param {DateTime} $1:dateFrom Start date to use for computing stats --- @param {String} $2:range Date range to use (year/month/week/day) -with score_table as (select date_trunc($2, created_at) as date1, - sum(CASE WHEN event_type = 'UPDATE_PROJET_SET_VISIBLE' THEN 1 ELSE -1 END) as score - from pfmv."Analytics" - WHERE event_type in ('UPDATE_PROJET_SET_VISIBLE', 'UPDATE_PROJET_SET_INVISIBLE') - and created_at >= $1 - group by 1 - order by 1 desc), - all_intervals as (SELECT date_trunc($2, ts) as date1 - FROM generate_series($1, now(), CONCAT('1 ',$2)::interval) AS ts - group by 1) -select all_intervals.date1::timestamp::date as "periode", coalesce(score, 0) as "score" -from score_table - right outer join all_intervals on all_intervals.date1 = score_table.date1 -order by 1; \ No newline at end of file diff --git a/src/app/api/stats/route.ts b/src/app/api/stats/route.ts index 2d7e6f81..0f7aab78 100644 --- a/src/app/api/stats/route.ts +++ b/src/app/api/stats/route.ts @@ -7,7 +7,6 @@ import { startOfWeek } from "date-fns/startOfWeek"; import { startOfDay } from "date-fns/startOfDay"; import { getNorthStarStats } from "@/src/lib/prisma/prisma-analytics-queries"; import { fr } from "date-fns/locale/fr"; -import { DateRange } from "@/src/helpers/dateUtils"; const StatsRouteSchema = z.object({ since: z @@ -27,11 +26,6 @@ type StatOutput = { stats: StatOutputRecord[]; }; -type GetNorthStarStatsProps = { - dateFrom: Date; - range: DateRange; -}; - export async function GET(request: NextRequest) { const parsedRequest = StatsRouteSchema.safeParse({ since: +(request.nextUrl.searchParams.get("since") ?? 0), @@ -42,21 +36,16 @@ export async function GET(request: NextRequest) { return NextResponse.json({ error: { message: "Invalid request", errors } }, { status: 400 }); } else { const { since: nbIntervals, periodicity } = parsedRequest.data; + + const ranges = { + year: startOfYear, + month: startOfMonth, + week: (date: Date) => startOfWeek(date, { weekStartsOn: 1, locale: fr }), + day: startOfDay, + }; + let dateBeginOfLastPeriod = new Date(); - switch (periodicity) { - case "year": - dateBeginOfLastPeriod = startOfYear(new Date()); - break; - case "month": - dateBeginOfLastPeriod = startOfMonth(new Date()); - break; - case "week": - dateBeginOfLastPeriod = startOfWeek(new Date(), { weekStartsOn: 1, locale: fr }); - break; - case "day": - dateBeginOfLastPeriod = startOfDay(new Date()); - break; - } + dateBeginOfLastPeriod = ranges[periodicity](new Date()); dateBeginOfLastPeriod = add(dateBeginOfLastPeriod, { minutes: -dateBeginOfLastPeriod.getTimezoneOffset(), }); @@ -67,69 +56,16 @@ export async function GET(request: NextRequest) { ...(periodicity === "week" && { weeks: 1 - nbIntervals }), ...(periodicity === "day" && { days: 1 - nbIntervals }), }); - const results = await getNorthStarStats(dateBeginOfFirstPeriod); - const computedStats = computeStats({ dateFrom: dateBeginOfFirstPeriod, range: periodicity }, results); - return NextResponse.json(computedStats); - } -} + const results = await getNorthStarStats({ dateFrom: dateBeginOfFirstPeriod, range: periodicity }); -const computeStats = ( - params: GetNorthStarStatsProps, - projets: { - created_at: Date; - }[], -): StatOutput => { - const statsMap = new Map(); - const now = new Date(); - let dateFrom = new Date(params.dateFrom); + const sanitizeResults: StatOutput = { + stats: results.map((result) => ({ + value: Number(result.score), + date: result.periode!, + })), + }; - while (dateFrom <= now) { - statsMap.set(dateFrom.toISOString(), 0); - switch (params.range) { - case "day": - dateFrom = add(dateFrom, { days: 1 }); - break; - case "week": - dateFrom = add(dateFrom, { weeks: 1 }); - break; - case "month": - dateFrom = add(dateFrom, { months: 1 }); - break; - case "year": - dateFrom = add(dateFrom, { years: 1 }); - break; - } + return NextResponse.json(sanitizeResults); } - - projets.forEach((projet) => { - let periodeDate: Date; - switch (params.range) { - case "day": - periodeDate = startOfDay(projet.created_at); - break; - case "week": - periodeDate = startOfWeek(projet.created_at, { weekStartsOn: 1, locale: fr }); - break; - case "month": - periodeDate = startOfMonth(projet.created_at); - break; - case "year": - periodeDate = startOfYear(projet.created_at); - break; - } - const periodeKey = periodeDate.toISOString(); - statsMap.set(periodeKey, (statsMap.get(periodeKey) || 0) + 1); - }); - - const stats = Array.from(statsMap.entries()) - .map(([periode, count]) => ({ - date: new Date(periode), - value: Number(count), - })) - .sort((a, b) => a.date.getTime() - b.date.getTime()); - - return { - stats, - }; -}; +} diff --git a/src/lib/prisma/prisma-analytics-queries.ts b/src/lib/prisma/prisma-analytics-queries.ts index a1705d8d..f889b9f9 100644 --- a/src/lib/prisma/prisma-analytics-queries.ts +++ b/src/lib/prisma/prisma-analytics-queries.ts @@ -1,9 +1,8 @@ /* eslint-disable max-len */ -import { Analytics } from "@prisma/client"; +import { Analytics, Prisma } from "@prisma/client"; import { prismaClient } from "./prismaClient"; import { DateRange } from "@/src/helpers/dateUtils"; -import { northStartStatQuery } from "@prisma/sql"; type AnalyticsProps = Omit; @@ -24,45 +23,33 @@ type GetNorthStarStatsProps = { range: DateRange; }; -export const getNorthStarStatsRaw = async (params: GetNorthStarStatsProps) => { - return prismaClient.$queryRawTyped(northStartStatQuery(params.dateFrom, params.range)); -}; - export const getNorthStarStats = async ( - dateFrom: GetNorthStarStatsProps["dateFrom"], -): Promise< - { - created_at: Date; - }[] -> => { - const projets = await prismaClient.projet.findMany({ - where: { - deleted_at: null, - created_at: { - gte: dateFrom, - }, - creator: { - NOT: [ - { - email: { - contains: "@ademe.fr", - }, - }, - { - email: { - contains: "@beta.gouv.fr", - }, - }, - { - email: "marie.racine@dihal.gouv.fr", - }, - ], - }, - }, - select: { - created_at: true, - }, - }); - - return projets; + params: GetNorthStarStatsProps, +): Promise<{ periode: Date; score: number }[]> => { + return prismaClient.$queryRaw( + Prisma.sql`with score_table as ( + select + date_trunc(${params.range}, p.created_at) as date1, + count(distinct p.id) as score + from pfmv."projet" p + join pfmv."user_projet" up on p.id = up.projet_id + join pfmv."User" u on up.user_id = u.id + WHERE p.created_at >= ${params.dateFrom} + and p.deleted_at IS NULL + and u.email NOT LIKE '%@ademe.fr' + and u.email NOT LIKE '%@beta.gouv.fr' + and u.email != 'marie.racine@dihal.gouv.fr' + group by 1 + ), + all_intervals as ( + SELECT date_trunc(${params.range}, ts) as date1 + FROM generate_series(${params.dateFrom}, now(), CONCAT('1 ', ${params.range})::interval) AS ts + ) + select + all_intervals.date1::timestamp::date as "periode", + coalesce(score, 0) as "score" + from score_table + right outer join all_intervals on all_intervals.date1 = score_table.date1 + order by 1;`, + ); }; diff --git a/src/types/sql.d.ts b/src/types/sql.d.ts deleted file mode 100644 index 755c736d..00000000 --- a/src/types/sql.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare module "@prisma/sql" { - export function northStartStatQuery(dateFrom: Date, range: string): string; -}