Skip to content

Commit

Permalink
Catch dispatch failures with task alert
Browse files Browse the repository at this point in the history
Signed-off-by: Aaron Chong <aaronchongth@gmail.com>
  • Loading branch information
aaronchongth committed Jan 30, 2024
1 parent 74d2ef9 commit 2b7f82e
Show file tree
Hide file tree
Showing 5 changed files with 40 additions and 3 deletions.
5 changes: 4 additions & 1 deletion packages/api-server/api_server/routes/internal.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,10 @@ async def process_msg(msg: Dict[str, Any], fleet_repo: FleetRepository) -> None:
await task_repo.save_task_state(task_state)
task_events.task_states.on_next(task_state)

if task_state.status == mdl.Status.completed:
if (
task_state.status == mdl.Status.completed
or task_state.status == mdl.Status.failed
):
alert = await alert_repo.create_alert(task_state.booking.id, "task")
if alert is not None:
alert_events.alerts.on_next(alert)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ def do(start_from: Optional[datetime]):
return

logger.info(f"Starting task {task.pk}")
logger.info(f"{task.task_request}")
asyncio.get_event_loop().create_task(run())
logger.warning(f"Schedule has {len(schedule.get_jobs())} jobs left")

Expand Down
2 changes: 2 additions & 0 deletions packages/api-server/api_server/routes/tasks/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
start_time_between_query,
)
from api_server.fast_io import FastIORouter, SubscriptionRequest
from api_server.logger import logger
from api_server.models.tortoise_models import TaskState as DbTaskState
from api_server.repositories import TaskRepository, task_repo_dep
from api_server.response import RawJSONResponse
Expand Down Expand Up @@ -201,6 +202,7 @@ async def post_dispatch_task(
resp = mdl.TaskDispatchResponse.parse_raw(
await tasks_service().call(request.json(exclude_none=True))
)
logger.info(resp)
if not resp.__root__.success:
return RawJSONResponse(resp.json(), 400)
new_state = cast(mdl.TaskDispatchResponseItem, resp.__root__).state
Expand Down
33 changes: 32 additions & 1 deletion packages/dashboard/src/components/tasks/task-alert.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import {
ApiServerModelsTortoiseModelsAlertsAlertLeaf,
LogEntry,
Status,
Status1,
TaskEventLog,
TaskState,
Tier,
Expand Down Expand Up @@ -55,6 +56,9 @@ export function TaskAlertDialog({ alert, removeAlert }: TaskAlertDialogProps): J
if (state.status && state.status === Status.Completed) {
return 'Task completed';
}
if (state.status && state.status === Status.Failed) {
return 'Task failed';
}
if (errorLogEntries.length !== 0) {
return 'Task error';
}
Expand Down Expand Up @@ -91,7 +95,24 @@ export function TaskAlertDialog({ alert, removeAlert }: TaskAlertDialogProps): J
},
];

// Second field would be any errors found
// Second field would be any dispatch errors
if (state.dispatch && state.dispatch.status === Status1.FailedToAssign) {
let errors = '';
if (state.dispatch.errors) {
for (const error of state.dispatch.errors) {
errors += `${JSON.stringify(error)}\n`;
}
}
content = [
...content,
{
title: 'Dispatch failure',
value: errors.length !== 0 ? errors : 'n/a',
},
];
}

// Third field would be any errors found
if (errorLogEntries.length !== 0) {
let consolidatedErrorMessages = '';
for (let entry of errorLogEntries) {
Expand Down Expand Up @@ -237,6 +258,16 @@ export function TaskAlertDialog({ alert, removeAlert }: TaskAlertDialogProps): J
late: task_late,
...logs,
});
} else if (state) {
setTaskAlert({
title: getAlertTitle(state, []),
progress: getTaskProgress(state),
content: getAlertContent(state, []),
color: getAlertColor(state, []),
acknowledgedBy: acknowledgedBy,
late: task_late,
task_id: state.booking.id,
});
}
})();
}, [rmf, alert.original_id, alert.acknowledged_by, alert.unix_millis_acknowledged_time]);
Expand Down
2 changes: 1 addition & 1 deletion packages/react-components/lib/alert-dialog.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ export const AlertDialog = React.memo((props: DialogAlertProps) => {
InputProps={{ readOnly: true, className: classes.textField }}
fullWidth={true}
multiline
maxRows={isScreenHeightLessThan800 ? 2 : 4}
maxRows={isScreenHeightLessThan800 ? 5 : 10}
margin="dense"
value={message.value}
/>
Expand Down

0 comments on commit 2b7f82e

Please sign in to comment.