Merge pull request #259 from calkit/ds-browse

Enable browsing through public datasets and replace figures page with learn page

backend/app/api/main.py

@@ -1,4 +1,4 @@
-from app.api.routes import login, misc, projects, users, orgs
+from app.api.routes import login, misc, projects, users, orgs, datasets
 from fastapi import APIRouter
 
 api_router = APIRouter()
@@ -7,3 +7,4 @@
 api_router.include_router(misc.router, tags=["misc"])
 api_router.include_router(projects.router, tags=["projects"])
 api_router.include_router(orgs.router, tags=["orgs"])
+api_router.include_router(datasets.router, tags=["datasets"])

backend/app/api/routes/datasets.py

@@ -0,0 +1,69 @@
+"""API endpoints for datasets."""
+
+import logging
+
+import sqlalchemy
+from app.api.deps import CurrentUser, SessionDep
+from app.models import Dataset, Project, ProjectPublic
+from fastapi import APIRouter
+from sqlmodel import SQLModel, func, or_, select
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+router = APIRouter()
+
+
+class DatasetResponse(SQLModel):
+    project: ProjectPublic
+    path: str
+    title: str | None
+    description: str | None
+    imported_from: str | None
+
+
+class DatasetsResponse(SQLModel):
+    data: list[DatasetResponse]
+    count: int
+
+
+@router.get("/datasets")
+def get_datasets(
+    session: SessionDep,
+    current_user: CurrentUser,
+    limit: int = 100,
+    offset: int = 0,
+    include_imported: bool = False,
+) -> DatasetsResponse:
+    # TODO: Handle collaborator access for private project datasets
+    count_query = (
+        select(func.count())
+        .select_from(Dataset)
+        .join(Project)
+        .where(
+            or_(
+                Project.is_public,
+                Project.owner_account_id == current_user.account.id,
+            )
+        )
+    )
+    if not include_imported:
+        count_query = count_query.filter(Dataset.imported_from.is_(None))
+    count = session.exec(count_query).one()
+    select_query = (
+        select(Dataset)
+        .join(Project)
+        .where(
+            or_(
+                Project.is_public,
+                Project.owner_account_id == current_user.account.id,
+            )
+        )
+        .order_by(sqlalchemy.asc(Project.title))
+        .limit(limit)
+        .offset(offset)
+    )
+    if not include_imported:
+        select_query = select_query.filter(Dataset.imported_from.is_(None))
+    datasets = session.exec(select_query).all()
+    return DatasetsResponse(data=datasets, count=count)

frontend/src/client/models.ts

@@ -134,6 +134,19 @@ export type DatasetDVCImport = {
   dvc_import?: DVCImport | null
 }
 
+export type DatasetResponse = {
+  project: ProjectPublic
+  path: string
+  title: string | null
+  description: string | null
+  imported_from: string | null
+}
+
+export type DatasetsResponse = {
+  data: Array<DatasetResponse>
+  count: number
+}
+
 export type DiscountCode = {
   id?: string
   created?: string

frontend/src/client/schemas.ts

@@ -615,6 +615,71 @@ export const $DatasetDVCImport = {
   },
 } as const
 
+export const $DatasetResponse = {
+  properties: {
+    project: {
+      type: "ProjectPublic",
+      isRequired: true,
+    },
+    path: {
+      type: "string",
+      isRequired: true,
+    },
+    title: {
+      type: "any-of",
+      contains: [
+        {
+          type: "string",
+        },
+        {
+          type: "null",
+        },
+      ],
+      isRequired: true,
+    },
+    description: {
+      type: "any-of",
+      contains: [
+        {
+          type: "string",
+        },
+        {
+          type: "null",
+        },
+      ],
+      isRequired: true,
+    },
+    imported_from: {
+      type: "any-of",
+      contains: [
+        {
+          type: "string",
+        },
+        {
+          type: "null",
+        },
+      ],
+      isRequired: true,
+    },
+  },
+} as const
+
+export const $DatasetsResponse = {
+  properties: {
+    data: {
+      type: "array",
+      contains: {
+        type: "DatasetResponse",
+      },
+      isRequired: true,
+    },
+    count: {
+      type: "number",
+      isRequired: true,
+    },
+  },
+} as const
+
 export const $DiscountCode = {
   properties: {
     id: {

frontend/src/client/services.ts

@@ -64,6 +64,7 @@ import type {
   OrgPost,
   OrgPublic,
   OrgSubscriptionUpdate,
+  DatasetsResponse,
 } from "./models"
 
 export type LoginData = {
@@ -392,6 +393,14 @@ export type OrgsData = {
   }
 }
 
+export type DatasetsData = {
+  GetDatasets: {
+    includeImported?: boolean
+    limit?: number
+    offset?: number
+  }
+}
+
 export class LoginService {
   /**
    * Login Access Token
@@ -2195,3 +2204,28 @@ export class OrgsService {
     })
   }
 }
+
+export class DatasetsService {
+  /**
+   * Get Datasets
+   * @returns DatasetsResponse Successful Response
+   * @throws ApiError
+   */
+  public static getDatasets(
+    data: DatasetsData["GetDatasets"] = {},
+  ): CancelablePromise<DatasetsResponse> {
+    const { limit = 100, offset = 0, includeImported = false } = data
+    return __request(OpenAPI, {
+      method: "GET",
+      url: "/datasets",
+      query: {
+        limit,
+        offset,
+        include_imported: includeImported,
+      },
+      errors: {
+        422: `Validation Error`,
+      },
+    })
+  }
+}

frontend/src/components/Common/Topbar.tsx

@@ -20,7 +20,7 @@ interface Props {
   children: React.ReactNode
 }
 
-const Links = ["Projects", "Datasets", "Software", "Figures"]
+const Links = ["Projects", "Datasets", "Learn"]
 
 const getPath = (link: React.ReactNode) => {
   const linkString = link?.toString()

frontend/src/routeTree.gen.ts

@@ -20,10 +20,9 @@ import { Route as LoginImport } from './routes/login'
 import { Route as CheckoutImport } from './routes/checkout'
 import { Route as LayoutImport } from './routes/_layout'
 import { Route as LayoutIndexImport } from './routes/_layout/index'
-import { Route as LayoutSoftwareImport } from './routes/_layout/software'
 import { Route as LayoutSettingsImport } from './routes/_layout/settings'
 import { Route as LayoutProjectsImport } from './routes/_layout/projects'
-import { Route as LayoutFiguresImport } from './routes/_layout/figures'
+import { Route as LayoutLearnImport } from './routes/_layout/learn'
 import { Route as LayoutDatasetsImport } from './routes/_layout/datasets'
 import { Route as LayoutAdminImport } from './routes/_layout/admin'
 import { Route as LayoutUserNameProjectNameLayoutImport } from './routes/_layout/$userName/$projectName/_layout'
@@ -82,11 +81,6 @@ const LayoutIndexRoute = LayoutIndexImport.update({
   getParentRoute: () => LayoutRoute,
 } as any)
 
-const LayoutSoftwareRoute = LayoutSoftwareImport.update({
-  path: '/software',
-  getParentRoute: () => LayoutRoute,
-} as any)
-
 const LayoutSettingsRoute = LayoutSettingsImport.update({
   path: '/settings',
   getParentRoute: () => LayoutRoute,
@@ -97,8 +91,8 @@ const LayoutProjectsRoute = LayoutProjectsImport.update({
   getParentRoute: () => LayoutRoute,
 } as any)
 
-const LayoutFiguresRoute = LayoutFiguresImport.update({
-  path: '/figures',
+const LayoutLearnRoute = LayoutLearnImport.update({
+  path: '/learn',
   getParentRoute: () => LayoutRoute,
 } as any)
 
@@ -249,11 +243,11 @@ declare module '@tanstack/react-router' {
       preLoaderRoute: typeof LayoutDatasetsImport
       parentRoute: typeof LayoutImport
     }
-    '/_layout/figures': {
-      id: '/_layout/figures'
-      path: '/figures'
-      fullPath: '/figures'
-      preLoaderRoute: typeof LayoutFiguresImport
+    '/_layout/learn': {
+      id: '/_layout/learn'
+      path: '/learn'
+      fullPath: '/learn'
+      preLoaderRoute: typeof LayoutLearnImport
       parentRoute: typeof LayoutImport
     }
     '/_layout/projects': {
@@ -270,13 +264,6 @@ declare module '@tanstack/react-router' {
       preLoaderRoute: typeof LayoutSettingsImport
       parentRoute: typeof LayoutImport
     }
-    '/_layout/software': {
-      id: '/_layout/software'
-      path: '/software'
-      fullPath: '/software'
-      preLoaderRoute: typeof LayoutSoftwareImport
-      parentRoute: typeof LayoutImport
-    }
     '/_layout/': {
       id: '/_layout/'
       path: '/'
@@ -384,10 +371,9 @@ export const routeTree = rootRoute.addChildren({
   LayoutRoute: LayoutRoute.addChildren({
     LayoutAdminRoute,
     LayoutDatasetsRoute,
-    LayoutFiguresRoute,
+    LayoutLearnRoute,
     LayoutProjectsRoute,
     LayoutSettingsRoute,
-    LayoutSoftwareRoute,
     LayoutIndexRoute,
     LayoutUserNameProjectNameRoute: LayoutUserNameProjectNameRoute.addChildren({
       LayoutUserNameProjectNameLayoutRoute:
@@ -434,10 +420,9 @@ export const routeTree = rootRoute.addChildren({
       "children": [
         "/_layout/admin",
         "/_layout/datasets",
-        "/_layout/figures",
+        "/_layout/learn",
         "/_layout/projects",
         "/_layout/settings",
-        "/_layout/software",
         "/_layout/",
         "/_layout/$userName/$projectName"
       ]
@@ -465,8 +450,8 @@ export const routeTree = rootRoute.addChildren({
       "filePath": "_layout/datasets.tsx",
       "parent": "/_layout"
     },
-    "/_layout/figures": {
-      "filePath": "_layout/figures.tsx",
+    "/_layout/learn": {
+      "filePath": "_layout/learn.tsx",
       "parent": "/_layout"
     },
     "/_layout/projects": {
@@ -477,10 +462,6 @@ export const routeTree = rootRoute.addChildren({
       "filePath": "_layout/settings.tsx",
       "parent": "/_layout"
     },
-    "/_layout/software": {
-      "filePath": "_layout/software.tsx",
-      "parent": "/_layout"
-    },
     "/_layout/": {
       "filePath": "_layout/index.tsx",
       "parent": "/_layout"