Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hammer/max downloads #909

Merged
merged 18 commits into from
Mar 11, 2024
Merged

Hammer/max downloads #909

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
32eb4ad
Use a limit of 1000 for all-task queries
aaronchongth Feb 28, 2024
f40b701
Raising limit to 5000
aaronchongth Feb 28, 2024
c2dae72
Added new API to get number of task states
aaronchongth Mar 7, 2024
35f61ff
First attempt at querying by chunks with a loading backdrop
aaronchongth Mar 7, 2024
6014ead
Count tasks based on query as well
aaronchongth Mar 8, 2024
1cc0fa2
Chunk-based query with global loading backdrop
aaronchongth Mar 8, 2024
9b68089
Query requests in chunks as well
aaronchongth Mar 8, 2024
16617a8
Clean up stale comments
aaronchongth Mar 8, 2024
40b91f0
Remove count endpoint, revert changes to API
aaronchongth Mar 8, 2024
0353728
Revert routes file
aaronchongth Mar 8, 2024
1de251a
Using normal pagination queries to query until the tasks run out, sti…
aaronchongth Mar 8, 2024
d911940
Use start time for query
aaronchongth Mar 8, 2024
18a721f
Use Button instead of IconButton for export and refresh
aaronchongth Mar 8, 2024
f5ed5e9
Remove stale import
aaronchongth Mar 8, 2024
d126767
Add a fixme for still querying for task requests during csv export
aaronchongth Mar 11, 2024
ac4a6c7
Rename variables more accurately since it will only query for the pas…
aaronchongth Mar 11, 2024
651562d
Changing download to export
aaronchongth Mar 11, 2024
efa1cd5
Moving task request query for only minimal export
aaronchongth Mar 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions packages/dashboard/src/components/app-base.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import {
Alert,
AlertProps,
Backdrop,
CircularProgress,
createTheme,
CssBaseline,
GlobalStyles,
Expand All @@ -14,6 +16,7 @@ import { loadSettings, saveSettings, Settings, ThemeMode } from '../settings';
import { AppController, AppControllerContext, SettingsContext } from './app-contexts';
import AppBar from './appbar';
import { AlertStore } from './alert-store';
import { AppEvents } from './app-events';
import { DeliveryAlertStore } from './delivery-alert-store';

const DefaultAlertDuration = 2000;
Expand All @@ -39,6 +42,7 @@ export function AppBase({ children }: React.PropsWithChildren<{}>): JSX.Element
const [alertMessage, setAlertMessage] = React.useState('');
const [alertDuration, setAlertDuration] = React.useState(DefaultAlertDuration);
const [extraAppbarIcons, setExtraAppbarIcons] = React.useState<React.ReactNode>(null);
const [openLoadingBackdrop, setOpenLoadingBackdrop] = React.useState(false);

const theme = React.useMemo(() => {
switch (settings.themeMode) {
Expand Down Expand Up @@ -70,10 +74,25 @@ export function AppBase({ children }: React.PropsWithChildren<{}>): JSX.Element
[updateSettings],
);

React.useEffect(() => {
const sub = AppEvents.loadingBackdrop.subscribe((value) => {
setOpenLoadingBackdrop(value);
});
return () => sub.unsubscribe();
}, []);

return (
<ThemeProvider theme={theme}>
<CssBaseline />
{settings.themeMode === ThemeMode.RmfDark && <GlobalStyles styles={rmfDarkLeaflet} />}
{openLoadingBackdrop && (
<Backdrop
sx={{ color: '#fff', zIndex: (theme) => theme.zIndex.drawer + 1 }}
open={openLoadingBackdrop}
>
<CircularProgress color="inherit" />
</Backdrop>
)}
<SettingsContext.Provider value={settings}>
<AppControllerContext.Provider value={appController}>
<AlertStore />
Expand Down
1 change: 1 addition & 0 deletions packages/dashboard/src/components/app-events.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,5 @@ export const AppEvents = {
levelSelect: new BehaviorSubject<Level | null>(null),
justLoggedIn: new BehaviorSubject<boolean>(false),
resetCamera: new Subject<[x: number, y: number, z: number, zoom: number]>(),
loadingBackdrop: new Subject<boolean>(),
};
140 changes: 93 additions & 47 deletions packages/dashboard/src/components/tasks/tasks-app.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import DownloadIcon from '@mui/icons-material/Download';
import RefreshIcon from '@mui/icons-material/Refresh';
import {
Box,
IconButton,
Button,
Menu,
MenuItem,
styled,
Expand Down Expand Up @@ -31,6 +31,7 @@ import { TaskSummary } from './task-summary';
import { downloadCsvFull, downloadCsvMinimal } from './utils';

const RefreshTaskQueueTableInterval = 15000;
const QueryLimit = 100;

enum TaskTablePanel {
QueueTable = 0,
Expand Down Expand Up @@ -240,64 +241,89 @@ export const TasksApp = React.memo(
selectedPanelIndex,
]);

const getAllTasks = async (timestamp: Date) => {
const getPastMonthTasks = async (timestamp: Date) => {
if (!rmf) {
return [];
}

const resp = await rmf.tasksApi.queryTaskStatesTasksGet(
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
`0,${timestamp.getTime()}`,
undefined,
undefined,
undefined,
undefined,
'-unix_millis_start_time',
undefined,
);
const allTasks = resp.data as TaskState[];
const currentMillis = timestamp.getTime();
const oneMonthMillis = 31 * 24 * 60 * 60 * 1000;
const allTasks: TaskState[] = [];
let queries: TaskState[] = [];
let queryIndex = 0;
do {
queries = (
await rmf.tasksApi.queryTaskStatesTasksGet(
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
undefined,
`${currentMillis - oneMonthMillis},${currentMillis}`,
undefined,
QueryLimit,
queryIndex * QueryLimit,
'-unix_millis_start_time',
undefined,
)
).data;
if (queries.length === 0) {
break;
}

allTasks.push(...queries);
queryIndex += 1;
} while (queries.length !== 0);
return allTasks;
};

const getAllTaskRequests = async (tasks: TaskState[]) => {
const getPastMonthTaskRequests = async (tasks: TaskState[]) => {
if (!rmf) {
return {};
}

const taskIds: string[] = tasks.map((task) => task.booking.id);
const taskIdsQuery = taskIds.join(',');
const taskRequests = (await rmf.tasksApi.queryTaskRequestsTasksRequestsGet(taskIdsQuery))
.data;

const taskRequestMap: Record<string, TaskRequest> = {};
let requestIndex = 0;
for (const id of taskIds) {
if (requestIndex < taskRequests.length && taskRequests[requestIndex]) {
taskRequestMap[id] = taskRequests[requestIndex];
const allTaskIds: string[] = tasks.map((task) => task.booking.id);
const queriesRequired = Math.ceil(allTaskIds.length / QueryLimit);
for (let i = 0; i < queriesRequired; i++) {
koonpeng marked this conversation as resolved.
Show resolved Hide resolved
const endingIndex = Math.min(allTaskIds.length, (i + 1) * QueryLimit);
const taskIds = allTaskIds.slice(i * QueryLimit, endingIndex);
const taskIdsQuery = taskIds.join(',');
const taskRequests = (await rmf.tasksApi.queryTaskRequestsTasksRequestsGet(taskIdsQuery))
.data;

let requestIndex = 0;
for (const id of taskIds) {
if (requestIndex < taskRequests.length && taskRequests[requestIndex]) {
taskRequestMap[id] = taskRequests[requestIndex];
}
++requestIndex;
}
++requestIndex;
}
return taskRequestMap;
};

const exportTasksToCsv = async (minimal: boolean) => {
AppEvents.loadingBackdrop.next(true);
const now = new Date();
const allTasks = await getAllTasks(now);
const allTaskRequests = await getAllTaskRequests(allTasks);
if (!allTasks || !allTasks.length) {
const pastMonthTasks = await getPastMonthTasks(now);

// FIXME: Task requests are currently required for parsing pickup and
// destination information. Once we start using TaskState.Booking.Labels
// to encode these information, we can skip querying for task requests.
const pastMonthTaskRequests = await getPastMonthTaskRequests(pastMonthTasks);
if (!pastMonthTasks || !pastMonthTasks.length) {
return;
}
if (minimal) {
downloadCsvMinimal(now, allTasks, allTaskRequests);
downloadCsvMinimal(now, pastMonthTasks, pastMonthTaskRequests);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. "download*" is misleading since it doesn't actually download, suggest to rename it to "save*" or add docs.
  2. It's weird that "minimal" contains info that is not in "full" (from what I see, pickup and dropoff), I will say to include the task request in full so it has the same data.
  3. Parsing the task request for pick and dropoff is also a red flag, it is out of scope of this PR so just add a FIXME. In future PR we can look at using task tags to store this information and just dump it, it also solves the issue of requiring double query to get all the required data.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. Changing it to export, 651562d

  2. Technically, full contains all the information of minimal and much more, since any number of pickups and dropoffs can be parsed from the task phases if needed. And since we will be adding pick and dropoff information in labels soon, I will not make any changes to this.

  3. FIXME was added previously d126767

Copy link
Collaborator

@koonpeng koonpeng Mar 11, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Technically, full contains all the information of minimal and much more, since any number of pickups and dropoffs can be parsed from the task phases if needed. And since we will be adding pick and dropoff information in labels soon, I will not make any changes to this.

Then we don't need to fetch task requests for "minimal"? Or do you mean that it will be included after we add the tags? If so then add a TODO.

} else {
downloadCsvFull(now, allTasks);
downloadCsvFull(now, pastMonthTasks);
}
AppEvents.loadingBackdrop.next(false);
};

const [anchorExportElement, setAnchorExportElement] = React.useState<null | HTMLElement>(
Expand All @@ -324,18 +350,27 @@ export const TasksApp = React.memo(
toolbar={
<Toolbar variant="dense">
<div>
<Tooltip title="Download" placement="top">
<IconButton
sx={{ marginBottom: isScreenHeightLessThan800 ? 1.8 : 0 }}
<Tooltip title="Export task history of the past 31 days" placement="top">
<Button
sx={{
fontSize: isScreenHeightLessThan800 ? '0.7rem' : 'inherit',
paddingTop: isScreenHeightLessThan800 ? 0 : 'inherit',
paddingBottom: isScreenHeightLessThan800 ? 0 : 'inherit',
marginBottom: isScreenHeightLessThan800 ? 1.8 : 'inherit',
}}
id="export-button"
aria-controls={openExportMenu ? 'export-menu' : undefined}
aria-haspopup="true"
aria-expanded={openExportMenu ? 'true' : undefined}
variant="outlined"
onClick={handleClickExportMenu}
color="inherit"
startIcon={
<DownloadIcon transform={`scale(${isScreenHeightLessThan800 ? 0.8 : 1})`} />
}
>
<DownloadIcon transform={`scale(${isScreenHeightLessThan800 ? 0.8 : 1})`} />
</IconButton>
Export past 31 days
</Button>
</Tooltip>
<Menu
id="export-menu"
Expand All @@ -348,34 +383,45 @@ export const TasksApp = React.memo(
>
<MenuItem
onClick={() => {
handleCloseExportMenu();
exportTasksToCsv(true);
handleCloseExportMenu();
}}
disableRipple
>
Export Minimal
</MenuItem>
<MenuItem
onClick={() => {
handleCloseExportMenu();
exportTasksToCsv(false);
handleCloseExportMenu();
}}
disableRipple
>
Export Full
</MenuItem>
</Menu>
</div>
<Tooltip title="Refresh" color="inherit" placement="top">
<IconButton
sx={{ marginBottom: isScreenHeightLessThan800 ? 1.8 : 0 }}
<Tooltip title="Refreshes the task queue table" color="inherit" placement="top">
<Button
sx={{
fontSize: isScreenHeightLessThan800 ? '0.7rem' : 'inherit',
paddingTop: isScreenHeightLessThan800 ? 0 : 'inherit',
paddingBottom: isScreenHeightLessThan800 ? 0 : 'inherit',
marginBottom: isScreenHeightLessThan800 ? 1.8 : 'inherit',
}}
id="refresh-button"
variant="outlined"
onClick={() => {
AppEvents.refreshTaskApp.next();
}}
aria-label="Refresh"
color="inherit"
startIcon={
<RefreshIcon transform={`scale(${isScreenHeightLessThan800 ? 0.8 : 1})`} />
}
>
<RefreshIcon transform={`scale(${isScreenHeightLessThan800 ? 0.8 : 1})`} />
</IconButton>
Refresh Task Queue
</Button>
</Tooltip>
</Toolbar>
}
Expand Down
Loading