From a1db318c64e8229672231d61ee8fff1fdbe109d7 Mon Sep 17 00:00:00 2001 From: Matthieu Date: Mon, 20 Jan 2025 13:17:19 +0000 Subject: [PATCH] feat: add data fact description to search --- adminSiteServer/apiRouter.ts | 10 +- adminSiteServer/apiRoutes/suggest.ts | 46 ++++ packages/@ourworldindata/utils/src/Util.ts | 2 +- site/search/ChartHit.scss | 45 +++ site/search/ChartHitsTextual.tsx | 305 +++++++++++++++++++++ site/search/SearchPanel.tsx | 12 +- 6 files changed, 416 insertions(+), 4 deletions(-) create mode 100644 site/search/ChartHitsTextual.tsx diff --git a/adminSiteServer/apiRouter.ts b/adminSiteServer/apiRouter.ts index 7a57a01899c..d7d027c1f73 100644 --- a/adminSiteServer/apiRouter.ts +++ b/adminSiteServer/apiRouter.ts @@ -63,7 +63,11 @@ import { handleDeleteChartRedirect, } from "./apiRoutes/redirects.js" import { triggerStaticBuild } from "./apiRoutes/routeUtils.js" -import { suggestGptTopics, suggestGptAltText } from "./apiRoutes/suggest.js" +import { + suggestGptTopics, + suggestGptAltText, + suggestDataPointDescription, +} from "./apiRoutes/suggest.js" import { handleGetFlatTagGraph, handlePostTagGraph, @@ -332,6 +336,10 @@ getRouteWithROTransaction( suggestGptAltText ) +apiRouter.post("/gpt/suggest-data-point-description", async (req) => ({ + description: await suggestDataPointDescription(req), +})) + // Tag graph routes getRouteWithROTransaction( apiRouter, diff --git a/adminSiteServer/apiRoutes/suggest.ts b/adminSiteServer/apiRoutes/suggest.ts index 4a294d43282..0c45d25ab8d 100644 --- a/adminSiteServer/apiRoutes/suggest.ts +++ b/adminSiteServer/apiRoutes/suggest.ts @@ -10,6 +10,8 @@ import { fetchGptGeneratedAltText } from "../imagesHelpers.js" import * as db from "../../db/db.js" import e from "express" import { Request } from "../authentication.js" +import OpenAI from "openai" +import { OPENAI_API_KEY } from "../../settings/serverSettings.js" export async function suggestGptTopics( req: Request, @@ -62,3 +64,47 @@ export async function suggestGptAltText( return { success: true, altText } } + +export async function suggestDataPointDescription( + req: Request +): Promise { + const { dataPoint, metadata } = req.body + if (!dataPoint || !metadata) throw new JsonError(`Invalid input data`, 400) + + let description: string | null = "" + try { + description = await fetchGptDataPointDescription(dataPoint, metadata) + } catch (error) { + console.error(`Error fetching GPT sentence`, error) + throw new JsonError(`Error fetching GPT sentence: ${error}`, 500) + } + + if (!description) { + throw new JsonError(`Unable to generate sentence`, 404) + } + + return description +} + +export async function fetchGptDataPointDescription( + dataPoint: any, + metadata: any +) { + const prompt = `Given the data point ${JSON.stringify(dataPoint)} and metadata ${JSON.stringify(metadata)}, + generate a data point fact. + - Do not add any information that is not directly supported by the data point and metadata. + - Do not prefix the fact with "Data point:" or similar.` + // console.log(prompt) + const openai = new OpenAI({ + apiKey: OPENAI_API_KEY, + }) + const completion = await openai.chat.completions.create({ + messages: [{ role: "user", content: prompt }], + model: "gpt-4o-mini", + }) + + const description = completion.choices[0]?.message?.content + if (!description) throw new JsonError("No response from GPT", 500) + // console.log("Generated Description:", description) + return description +} diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts index 5209f0e6702..267168ae600 100644 --- a/packages/@ourworldindata/utils/src/Util.ts +++ b/packages/@ourworldindata/utils/src/Util.ts @@ -599,7 +599,7 @@ export const fetchText = async (url: string): Promise => { const _getUserCountryInformation = async (): Promise< UserCountryInformation | undefined > => - await fetchWithRetry("/detect-country") + await fetchWithRetry("https://detect-country.owid.io") .then((res) => res.json()) .then((res) => res.country) .catch(() => undefined) diff --git a/site/search/ChartHit.scss b/site/search/ChartHit.scss index a319d9b2bd5..1fb7dbc1d23 100644 --- a/site/search/ChartHit.scss +++ b/site/search/ChartHit.scss @@ -95,3 +95,48 @@ svg.chart-hit-icon { } } } + +.chart-hit-textual { + // border-bottom: 1px solid $gray-20; + padding-bottom: 24px; +} + +.chart-hit-textual__description { + margin-top: 32px; + font-size: 1.5rem; + a { + @include owid-link-90; + text-decoration: none; + &:visited { + color: $blue-90; + } + &:hover { + text-decoration: underline; + } + } +} + +.chart-hit-textual__debug { + display: flex; + gap: 0.5em; + margin-top: 4px; + justify-content: end; +} + +.chart-hit-textual__get-data { + a { + @include owid-link-60; + &:visited { + color: $blue-60; + } + } +} + +.chart-hit-textual__data { + color: $gray-60; + font-size: 0.8rem; + + a { + @include owid-link-60; + } +} diff --git a/site/search/ChartHitsTextual.tsx b/site/search/ChartHitsTextual.tsx new file mode 100644 index 00000000000..0fcfed9b2ab --- /dev/null +++ b/site/search/ChartHitsTextual.tsx @@ -0,0 +1,305 @@ +import { useEffect, useMemo, useState } from "react" +import { + debounce, + formatValue, + getUserCountryInformation, + OwidVariableMixedData, + OwidVariableWithSourceAndDimension, + pick, + Region, +} from "@ourworldindata/utils" +import { ChartRecordType, IChartHit } from "./searchTypes.js" +import { getEntityQueryStr, pickEntitiesForChartHit } from "./SearchUtils.js" +import { HitAttributeHighlightResult } from "instantsearch.js" +import { + BAKED_BASE_URL, + BAKED_GRAPHER_URL, + DATA_API_URL, +} from "../../settings/clientSettings.js" +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome/index.js" +import { faMapMarkerAlt } from "@fortawesome/free-solid-svg-icons" +import { useHits, UseHitsProps } from "react-instantsearch" +import { EXPLORERS_ROUTE_FOLDER } from "@ourworldindata/explorer" + +export function ChartHitsTextual({ + searchQueryRegionsMatches, + ...hitProps +}: { + searchQueryRegionsMatches?: Region[] | undefined +} & UseHitsProps) { + const { items } = useHits(hitProps) + if (items.length === 0) return null + return ( + + ) +} + +const ChartHitTextual = ({ + hit, + searchQueryRegionsMatches, +}: { + hit: IChartHit + searchQueryRegionsMatches?: Region[] | undefined +}) => { + const isExplorerView = hit.type === ChartRecordType.ExplorerView + + const entities = useMemo( + () => + pickEntitiesForChartHit( + hit._highlightResult?.availableEntities as + | HitAttributeHighlightResult[] + | undefined, + hit.availableEntities, + searchQueryRegionsMatches + ), + [ + hit._highlightResult?.availableEntities, + hit.availableEntities, + searchQueryRegionsMatches, + ] + ) + const entityQueryStr = useMemo( + () => getEntityQueryStr(entities), + [entities] + ) + + const fullQueryParams = isExplorerView + ? hit.queryParams! + entityQueryStr.replace("?", "&") + : entityQueryStr + + const chartUrl = isExplorerView + ? `${BAKED_BASE_URL}/${EXPLORERS_ROUTE_FOLDER}/${hit.slug}${fullQueryParams}` + : `${BAKED_GRAPHER_URL}/${hit.slug}${fullQueryParams}` + + const [localRegionCodes, setLocalRegionCodes] = useState([ + "OWID_WRL", + ]) + const [data, setData] = useState(null) + const [metadata, setMetadata] = + useState(null) + // const [date, setDate] = useState(undefined) + // const [value, setValue] = useState(undefined) + const [formattedValue, setFormattedValue] = useState( + undefined + ) + const [formattedDate, setFormattedDate] = useState( + undefined + ) + const [gptDescription, setGptDescription] = useState(null) + + useEffect(() => { + const fetchDataAndMetadata = async () => { + try { + const response = await fetch(chartUrl) + const text = await response.text() + const parser = new DOMParser() + const doc = parser.parseFromString(text, "text/html") + const dataLinkTag = doc.querySelector( + `link[href^="${DATA_API_URL}"][href$=".data.json"]` + ) + const metadataLinkTag = doc.querySelector( + `link[href^="${DATA_API_URL}"][href$=".metadata.json"]` + ) + if (!dataLinkTag || !metadataLinkTag) return + + const dataUrl = dataLinkTag.getAttribute("href") + const metadataUrl = metadataLinkTag.getAttribute("href") + // console.log("Data URL:", dataUrl) + // console.log("Metadata URL:", metadataUrl) + if (!dataUrl || !metadataUrl) return + + const dataResponse = await fetch(dataUrl) + const jsonData = await dataResponse.json() + // console.log("Chart Data JSON:", jsonData) + const metadataResponse = await fetch(metadataUrl) + const metadataJson = await metadataResponse.json() + // console.log("Metadata JSON:", metadataJson) + setData(jsonData) + setMetadata(metadataJson) + } catch (error) { + console.error("Error fetching chart data:", error) + } + } + + const debouncedFetchDataAndMetadata = debounce( + fetchDataAndMetadata, + 250 + ) + void debouncedFetchDataAndMetadata() + + return () => { + debouncedFetchDataAndMetadata.cancel() + } + }, [chartUrl]) + + useEffect(() => { + const detectLocalRegion = async (): Promise => { + try { + const localCountryInfo = await getUserCountryInformation() + // console.log("Local Country Info:", localCountryInfo) + if (!localCountryInfo) { + return + } + + const localRegionCodes = [ + localCountryInfo.code, + ...(localCountryInfo.regions ?? []), + "OWID_WRL", + ] + setLocalRegionCodes(localRegionCodes) + } catch { + console.error("Error detecting local region") + } + } + void detectLocalRegion() + }, []) + + const getEntityIdByRegionCode = ( + code: string + ): [number, string | undefined] | undefined => { + const entity = metadata?.dimensions.entities.values.find( + (entity) => entity.code === code + ) + if (!entity) return + return [entity.id, entity.name] + } + + const getValueForEntity = ( + entityId?: number, + year?: number + ): [number, number | string] | [] => { + if (!data || !entityId) return [] + const entityIndices = data.entities.reduce( + (indices, e, i) => { + if (e === entityId) indices.push(i) + return indices + }, + [] + ) + + if (entityIndices.length === 0) return [] + + const targetIndex = + year !== undefined + ? (entityIndices.find((i) => data.years[i] === year) ?? + entityIndices[entityIndices.length - 1]) + : entityIndices[entityIndices.length - 1] + + const date = data.years[targetIndex] + const value = data.values[targetIndex] + return [date, value] + } + // TODO handle multiple local regions + const [entityId, entityName] = + getEntityIdByRegionCode(localRegionCodes[0]) || [] + + // console.log("Entity ID:", entityId) + const [date, value] = getValueForEntity(entityId) + + //TODO: remove effect + useEffect(() => { + if (value !== undefined && metadata) { + setFormattedValue( + typeof value === "number" + ? formatValue(value, { ...metadata.display }) + : value + ) + } + if ( + metadata?.display?.yearIsDay && + metadata.display.zeroDay && + date !== undefined + ) { + // Special case for when the year is actually an offset in days (e.g. for COVID-19 data) + const zeroDay = new Date(metadata.display.zeroDay) + const offsetDate = new Date(zeroDay) + offsetDate.setDate(zeroDay.getDate() + date) + setFormattedDate(offsetDate.toISOString().split("T")[0]) + } else { + setFormattedDate(date?.toString()) + } + }, [date, value, metadata]) + + useEffect(() => { + const generateDataPointDescription = async () => { + if (!formattedDate || !formattedValue || !metadata) return + try { + const response = await fetch( + "/admin/api/gpt/suggest-data-point-description", + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + dataPoint: { + entity: entityName, + formattedValue, + formattedDate, + }, + metadata: pick(metadata, [ + "descriptionShort", + "name", + "display", + ]), + }), + } + ) + const result = await response.json() + setGptDescription(result.description) + } catch (error) { + console.error("Error generating data point description:", error) + } + } + // console.log("generateDataPointDescription") + void generateDataPointDescription() + }, [formattedValue, formattedDate, metadata, entityName]) + + return ( + <> + {gptDescription && ( +
+ +
+
+ + {/*
+ Date: {date} +
+
+ Name: {metadata?.descriptionShort || metadata?.name} +
+
+ Value: {formattedValue} +
*/} +
+ Report +
+
+
+ )} + {/* {entities.length > 0 && ( +
    + {entities.map((entity, i) => ( +
  • + {i === 0 && ( + + )} + {entity} +
    • + ))} +
+ )} */} + + ) +} diff --git a/site/search/SearchPanel.tsx b/site/search/SearchPanel.tsx index bf18e0a5023..cadeb31a62f 100644 --- a/site/search/SearchPanel.tsx +++ b/site/search/SearchPanel.tsx @@ -58,6 +58,7 @@ import { getEntityQueryStr, } from "./SearchUtils.js" import { ChartHit } from "./ChartHit.js" +import { ChartHitsTextual } from "./ChartHitsTextual.js" const siteAnalytics = new SiteAnalytics() @@ -553,6 +554,13 @@ const SearchResults = (props: SearchResultsProps) => { className="search-results" data-active-filter={activeCategoryFilter} > + + + + + {/* This is using the InstantSearch index specified in InstantSearchContainer */} - + /> */}