From 5cb15353ed1dee6377c2e831cc8caca36ca3d295 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 20 Nov 2024 12:19:15 +0100 Subject: [PATCH 1/5] Admin Oveview API --- server/mergin/auth/api.yaml | 47 ++++++++++++++++++++++++++++++++ server/mergin/auth/controller.py | 25 +++++++++++++++-- server/mergin/sync/utils.py | 35 +++++++++++++++++++++++- server/mergin/sync/workspace.py | 4 +-- 4 files changed, 105 insertions(+), 6 deletions(-) diff --git a/server/mergin/auth/api.yaml b/server/mergin/auth/api.yaml index f89e110d..4cee2238 100644 --- a/server/mergin/auth/api.yaml +++ b/server/mergin/auth/api.yaml @@ -311,6 +311,28 @@ paths: $ref: "#/components/responses/Forbidden" "404": $ref: "#/components/responses/NotFoundResp" + /app/admin/usage: + get: + tags: + - user + - admin + - project + summary: Get server usage statistics + description: List server storage, projects, active contributors for admin + operationId: mergin.auth.controller.get_server_usage + responses: + "200": + description: Server usage details + content: + application/json: + schema: + $ref: "#/components/schemas/ServerUsage" + "400": + $ref: "#/components/responses/BadStatusResp" + "401": + $ref: "#/components/responses/UnauthorizedError" + "403": + $ref: "#/components/responses/Forbidden" /app/auth/login: post: summary: Login @@ -845,3 +867,28 @@ components: - reader - guest example: reader + ServerUsage: + type: object + properties: + active_monthly_contributors: + type: array + description: count of users who made a project change last months + items: + type: integer + example: 2 + projects: + type: integer + description: total number of projects + example: 12 + storage: + type: string + description: projest files size in bytes + example: 1024 kB + users: + type: integer + description: count of registered accounts + example: 6 + workspaces: + type: integer + description: number of workspaces on the server + example: 3 diff --git a/server/mergin/auth/controller.py b/server/mergin/auth/controller.py index cdea3181..3b33cadc 100644 --- a/server/mergin/auth/controller.py +++ b/server/mergin/auth/controller.py @@ -6,7 +6,7 @@ import pytz from datetime import datetime, timedelta from connexion import NoContent -from sqlalchemy import func, desc, asc, or_ +from sqlalchemy import func, desc, asc from sqlalchemy.sql.operators import is_ from flask import request, current_app, jsonify, abort, render_template from flask_login import login_user, logout_user, current_user @@ -35,8 +35,9 @@ UserChangePasswordForm, ApiLoginForm, ) -from ..app import DEPRECATION_API_MSG, db -from ..utils import format_time_delta +from ..app import db +from ..sync.models import Project +from ..sync.utils import files_size # public endpoints @@ -498,3 +499,21 @@ def get_user_info(): for inv in invitations ] return user_info, 200 + + +@auth_required(permissions=["admin"]) +def get_server_usage(): + data = { + "active_monthly_contributors": + [ + current_app.ws_handler.monthly_contributors_count(), + current_app.ws_handler.monthly_contributors_count(month_offset=1), + current_app.ws_handler.monthly_contributors_count(month_offset=2), + current_app.ws_handler.monthly_contributors_count(month_offset=3), + ], + "projects": Project.query.count(), + "storage": files_size(), + "users": User.query.count(), + "workspaces": current_app.ws_handler.workspace_count(), + } + return data, 200 diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index 724b0c31..e8c9f367 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -9,12 +9,12 @@ import secrets from threading import Timer from uuid import UUID -from connexion import NoContent from shapely import wkb from shapely.errors import ShapelyError from gevent import sleep from flask import Request from typing import Optional +from sqlalchemy import text def generate_checksum(file, chunk_size=4096): @@ -304,3 +304,36 @@ def split_project_path(project_path): def get_device_id(request: Request) -> Optional[str]: """Get device uuid from http header X-Device-Id""" return request.headers.get("X-Device-Id") + + +def files_size(): + """Get total size of all files""" + from mergin.app import db + + files_size = text( + f""" + WITH partials AS ( + WITH latest_files AS ( + SELECT distinct unnest(file_history_ids) AS file_id + FROM latest_project_files pf + ) + SELECT + SUM(size) + FROM file_history + WHERE change = 'create'::push_change_type OR change = 'update'::push_change_type + UNION + SELECT + SUM(COALESCE((diff ->> 'size')::bigint, 0)) + FROM file_history + WHERE change = 'update_diff'::push_change_type + UNION + SELECT + SUM(size) + FROM latest_files lf + LEFT OUTER JOIN file_history fh ON fh.id = lf.file_id + WHERE fh.change = 'update_diff'::push_change_type + ) + SELECT pg_size_pretty(SUM(sum)) FROM partials; + """ + ) + return db.session.execute(files_size).scalar() diff --git a/server/mergin/sync/workspace.py b/server/mergin/sync/workspace.py index 5b5ac84e..0cd835e1 100644 --- a/server/mergin/sync/workspace.py +++ b/server/mergin/sync/workspace.py @@ -255,7 +255,7 @@ def workspace_count(): return 1 @staticmethod - def monthly_contributors_count(): + def monthly_contributors_count(month_offset=0): today = datetime.now(timezone.utc) year = today.year month = today.month @@ -263,7 +263,7 @@ def monthly_contributors_count(): db.session.query(ProjectVersion.author_id) .filter( extract("year", ProjectVersion.created) == year, - extract("month", ProjectVersion.created) == month, + extract("month", ProjectVersion.created) == month - month_offset, ) .group_by(ProjectVersion.author_id) .count() From 8b164b7e4f86654db938b2db4f9223715fde7c19 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 20 Nov 2024 12:20:00 +0100 Subject: [PATCH 2/5] Cleanup --- server/mergin/sync/public_api_controller.py | 1 - .../community/1ab5b02ce532_version_author_name_to_user_id.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 2cf62ee0..3a28b47e 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -12,7 +12,6 @@ from typing import Dict from urllib.parse import quote import uuid -from time import time from datetime import datetime import psycopg2 from blinker import signal diff --git a/server/migrations/community/1ab5b02ce532_version_author_name_to_user_id.py b/server/migrations/community/1ab5b02ce532_version_author_name_to_user_id.py index 3dfbadb4..93747d70 100644 --- a/server/migrations/community/1ab5b02ce532_version_author_name_to_user_id.py +++ b/server/migrations/community/1ab5b02ce532_version_author_name_to_user_id.py @@ -1,7 +1,7 @@ """Migrage project version author name to user.id Revision ID: 1ab5b02ce532 -Revises: 57d0de13ce4a +Revises: 1c23e3be03a3 Create Date: 2024-09-06 14:01:40.668483 """ From 928fb81224e3575d9877c4f6d7725e2bdbb96588 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 21 Nov 2024 17:12:17 +0100 Subject: [PATCH 3/5] Add Overview page to Admin --- .../src/modules/layout/components/Sidebar.vue | 6 + web-app/packages/admin-app/src/router.ts | 15 ++- web-app/packages/admin-lib/components.d.ts | 1 + .../admin-lib/src/modules/admin/adminApi.ts | 10 +- .../admin/components/AdminProjectsTable.vue | 4 +- .../src/modules/admin/components/index.ts | 1 + .../admin-lib/src/modules/admin/routes.ts | 1 + .../admin-lib/src/modules/admin/store.ts | 15 ++- .../admin-lib/src/modules/admin/types.ts | 11 ++ .../src/modules/admin/views/OverviewView.vue | 124 ++++++++++++++++++ .../src/modules/admin/views/index.ts | 1 + web-app/packages/admin-lib/src/shims-vue.d.ts | 2 + .../lib/src/common/components/Card.vue | 35 +++++ .../lib/src/common/components/index.ts | 1 + 14 files changed, 218 insertions(+), 9 deletions(-) create mode 100644 web-app/packages/admin-lib/src/modules/admin/views/OverviewView.vue create mode 100644 web-app/packages/lib/src/common/components/Card.vue diff --git a/web-app/packages/admin-app/src/modules/layout/components/Sidebar.vue b/web-app/packages/admin-app/src/modules/layout/components/Sidebar.vue index 17e52b4b..44f63887 100644 --- a/web-app/packages/admin-app/src/modules/layout/components/Sidebar.vue +++ b/web-app/packages/admin-app/src/modules/layout/components/Sidebar.vue @@ -13,6 +13,12 @@ import { useRoute } from 'vue-router' const route = useRoute() const sidebarItems = computed(() => [ + { + title: 'Overview', + to: '/overview', + icon: 'ti ti-layout-dashboard', + active: route.matched.some((item) => item.name === AdminRoutes.OVERVIEW) + }, { title: 'Accounts', to: '/accounts', diff --git a/web-app/packages/admin-app/src/router.ts b/web-app/packages/admin-app/src/router.ts index 795d43d6..d4a423cf 100644 --- a/web-app/packages/admin-app/src/router.ts +++ b/web-app/packages/admin-app/src/router.ts @@ -12,7 +12,8 @@ import { ProjectFilesView, ProjectSettingsView, ProjectVersionView, - ProjectVersionsView + ProjectVersionsView, + OverviewView } from '@mergin/admin-lib' import { NotFoundView, @@ -48,7 +49,7 @@ export const createRouter = (pinia: Pinia) => { { path: '/', name: 'admin', - redirect: '/accounts' + redirect: '/overview' }, { path: '/accounts', @@ -144,6 +145,16 @@ export const createRouter = (pinia: Pinia) => { header: AppHeader }, props: true + }, + { + path: '/overview', + name: AdminRoutes.OVERVIEW, + components: { + default: OverviewView, + sidebar: Sidebar, + header: AppHeader + }, + props: true } ] }) diff --git a/web-app/packages/admin-lib/components.d.ts b/web-app/packages/admin-lib/components.d.ts index e548e1e2..89889c60 100644 --- a/web-app/packages/admin-lib/components.d.ts +++ b/web-app/packages/admin-lib/components.d.ts @@ -15,6 +15,7 @@ declare module 'vue' { PInputSwitch: typeof import('primevue/inputswitch')['default'] PInputText: typeof import('primevue/inputtext')['default'] PPassword: typeof import('primevue/password')['default'] + PProgressBar: typeof import('primevue/progressbar')['default'] PTabPanel: typeof import('primevue/tabpanel')['default'] PTabView: typeof import('primevue/tabview')['default'] RouterLink: typeof import('vue-router')['RouterLink'] diff --git a/web-app/packages/admin-lib/src/modules/admin/adminApi.ts b/web-app/packages/admin-lib/src/modules/admin/adminApi.ts index f9e52e23..6380ef6c 100644 --- a/web-app/packages/admin-lib/src/modules/admin/adminApi.ts +++ b/web-app/packages/admin-lib/src/modules/admin/adminApi.ts @@ -3,8 +3,6 @@ // SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial import { - ApiRequestSuccessInfo, - errorUtils, LoginData, PaginatedUsersParams, UserProfileResponse, @@ -14,13 +12,13 @@ import { AxiosResponse } from 'axios' import { AdminModule } from '@/modules/admin/module' import { - UsersParams, UpdateUserData, UsersResponse, LatestServerVersionResponse, CreateUserData, PaginatedAdminProjectsResponse, - PaginatedAdminProjectsParams + PaginatedAdminProjectsParams, + ServerUsageResponse } from '@/modules/admin/types' export const AdminApi = { @@ -98,5 +96,9 @@ export const AdminApi = { `/app/project/removed-project/restore/${id}`, null ) + }, + + async getServerUsage(): Promise> { + return AdminModule.httpService.get('/app/admin/usage', ) } } diff --git a/web-app/packages/admin-lib/src/modules/admin/components/AdminProjectsTable.vue b/web-app/packages/admin-lib/src/modules/admin/components/AdminProjectsTable.vue index fa114c9a..8ac2ea4f 100644 --- a/web-app/packages/admin-lib/src/modules/admin/components/AdminProjectsTable.vue +++ b/web-app/packages/admin-lib/src/modules/admin/components/AdminProjectsTable.vue @@ -70,9 +70,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial :sortable="header.sortable" > + + + + + diff --git a/web-app/packages/admin-lib/src/modules/admin/views/index.ts b/web-app/packages/admin-lib/src/modules/admin/views/index.ts index d24ec2fd..cf67da4e 100644 --- a/web-app/packages/admin-lib/src/modules/admin/views/index.ts +++ b/web-app/packages/admin-lib/src/modules/admin/views/index.ts @@ -12,3 +12,4 @@ export { default as ProjectSettingsView } from './ProjectSettingsView.vue' export { default as ProjectFilesView } from './ProjectFilesView.vue' export { default as ProjectVersionView } from './ProjectVersionView.vue' export { default as ProjectVersionsView } from './ProjectVersionsView.vue' +export { default as OverviewView } from './OverviewView.vue' diff --git a/web-app/packages/admin-lib/src/shims-vue.d.ts b/web-app/packages/admin-lib/src/shims-vue.d.ts index 3642a896..bce55c3d 100644 --- a/web-app/packages/admin-lib/src/shims-vue.d.ts +++ b/web-app/packages/admin-lib/src/shims-vue.d.ts @@ -5,6 +5,7 @@ import { ComponentCustomPropertyFilters } from '@mergin/lib' import { MerginComponentUuid } from './modules/form/types' +import {Router} from "vue-router"; declare module '*.vue' { @@ -18,5 +19,6 @@ declare module '@vue/runtime-core' { interface ComponentCustomProperties { $filters: ComponentCustomPropertyFilters merginComponentUuid: MerginComponentUuid + $router: Router } } diff --git a/web-app/packages/lib/src/common/components/Card.vue b/web-app/packages/lib/src/common/components/Card.vue new file mode 100644 index 00000000..af0304a8 --- /dev/null +++ b/web-app/packages/lib/src/common/components/Card.vue @@ -0,0 +1,35 @@ + + + + + diff --git a/web-app/packages/lib/src/common/components/index.ts b/web-app/packages/lib/src/common/components/index.ts index c3fbe448..a3632cff 100644 --- a/web-app/packages/lib/src/common/components/index.ts +++ b/web-app/packages/lib/src/common/components/index.ts @@ -18,3 +18,4 @@ export { default as AppOnboardingPage } from './AppOnboardingPage.vue' export * from './types' export * from './data-view' export * from './app-settings' +export { default as OverviewCard } from './Card.vue' From 8436f36ed7abaa89fec1ca62262fde53749c23e5 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 26 Nov 2024 11:03:12 +0100 Subject: [PATCH 4/5] add missing UsageCards --- .../src/modules/admin/components/index.ts | 1 - .../src/modules/admin/views/OverviewView.vue | 197 +++++++++++------- .../components/{Card.vue => UsageCard.vue} | 0 .../lib/src/common/components/index.ts | 2 +- 4 files changed, 122 insertions(+), 78 deletions(-) rename web-app/packages/lib/src/common/components/{Card.vue => UsageCard.vue} (100%) diff --git a/web-app/packages/admin-lib/src/modules/admin/components/index.ts b/web-app/packages/admin-lib/src/modules/admin/components/index.ts index 79f32edd..f831d33a 100644 --- a/web-app/packages/admin-lib/src/modules/admin/components/index.ts +++ b/web-app/packages/admin-lib/src/modules/admin/components/index.ts @@ -8,4 +8,3 @@ export { default as CreateUserForm } from './CreateUserForm.vue' export { default as ServerNotConfigured } from './ServerNotConfigured.vue' export { default as AdminProjectsTable } from './AdminProjectsTable.vue' export { default as AccountsTable } from './AccountsTable.vue' -export { default as OverviewCard } from '../../../../../lib/src/common/components/Card.vue' diff --git a/web-app/packages/admin-lib/src/modules/admin/views/OverviewView.vue b/web-app/packages/admin-lib/src/modules/admin/views/OverviewView.vue index 69ec5743..6c9fbbfc 100644 --- a/web-app/packages/admin-lib/src/modules/admin/views/OverviewView.vue +++ b/web-app/packages/admin-lib/src/modules/admin/views/OverviewView.vue @@ -5,99 +5,146 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial -->