diff --git a/frontend/src/libs/run.ts b/frontend/src/libs/run.ts index e49e4c28f..b1a626bf8 100644 --- a/frontend/src/libs/run.ts +++ b/frontend/src/libs/run.ts @@ -39,7 +39,11 @@ export const getStatusIconType = ( export const getStatusIconColor = ( status: IRun['status'] | TJobStatus, terminationReason: string | null | undefined, + statusMessage: string, ): StatusIndicatorProps.Color | undefined => { + if (statusMessage === 'No fleets') { + return 'red'; + } if (terminationReason === 'failed_to_start_due_to_no_capacity' || terminationReason === 'interrupted_by_no_capacity') { return 'yellow'; } diff --git a/frontend/src/pages/Runs/Details/RunDetails/index.tsx b/frontend/src/pages/Runs/Details/RunDetails/index.tsx index 1547fa886..bdf8484b1 100644 --- a/frontend/src/pages/Runs/Details/RunDetails/index.tsx +++ b/frontend/src/pages/Runs/Details/RunDetails/index.tsx @@ -62,6 +62,8 @@ export const RunDetails = () => { const finishedAt = getRunListFinishedAt(runData); + const statusMessage = getRunStatusMessage(runData); + return ( <> {t('common.general')}}> @@ -112,9 +114,9 @@ export const RunDetails = () => {
- {getRunStatusMessage(runData)} + {statusMessage}
diff --git a/frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx b/frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx index 9f0514342..285c29ad9 100644 --- a/frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx +++ b/frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx @@ -84,13 +84,14 @@ export const useColumnsDefinitions = () => { const terminationReason = finishedRunStatuses.includes(item.status) ? item.latest_job_submission?.termination_reason : null; + const statusMessage = getRunStatusMessage(item); return ( - {getRunStatusMessage(item)} + {statusMessage} ); }, diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py index f942ca05b..d025160d0 100644 --- a/src/dstack/_internal/cli/services/configurators/run.py +++ b/src/dstack/_internal/cli/services/configurators/run.py @@ -106,7 +106,12 @@ def apply_configuration( ssh_identity_file=configurator_args.ssh_identity_file, ) - print_run_plan(run_plan, max_offers=configurator_args.max_offers) + no_fleets = False + if len(run_plan.job_plans[0].offers) == 0: + if len(self.api.client.fleets.list(self.api.project)) == 0: + no_fleets = True + + print_run_plan(run_plan, max_offers=configurator_args.max_offers, no_fleets=no_fleets) confirm_message = "Submit a new run?" if conf.name: diff --git a/src/dstack/_internal/cli/utils/common.py b/src/dstack/_internal/cli/utils/common.py index c75f08b81..a7f6dfd6d 100644 --- a/src/dstack/_internal/cli/utils/common.py +++ b/src/dstack/_internal/cli/utils/common.py @@ -32,6 +32,12 @@ " https://dstack.ai/docs/guides/troubleshooting/#no-offers" "[/]\n" ) +NO_FLEETS_WARNING = ( + "[warning]" + "The project has no fleets. Create one before submitting a run:" + " https://dstack.ai/docs/concepts/fleets" + "[/]\n" +) def cli_error(e: DstackError) -> CLIError: diff --git a/src/dstack/_internal/cli/utils/run.py b/src/dstack/_internal/cli/utils/run.py index 68dc828f7..1b6dfbaed 100644 --- a/src/dstack/_internal/cli/utils/run.py +++ b/src/dstack/_internal/cli/utils/run.py @@ -6,7 +6,12 @@ from dstack._internal.cli.models.offers import OfferCommandOutput, OfferRequirements from dstack._internal.cli.models.runs import PsCommandOutput -from dstack._internal.cli.utils.common import NO_OFFERS_WARNING, add_row_from_dict, console +from dstack._internal.cli.utils.common import ( + NO_FLEETS_WARNING, + NO_OFFERS_WARNING, + add_row_from_dict, + console, +) from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.configurations import DevEnvironmentConfiguration from dstack._internal.core.models.instances import ( @@ -75,7 +80,10 @@ def print_runs_json(project: str, runs: List[Run]) -> None: def print_run_plan( - run_plan: RunPlan, max_offers: Optional[int] = None, include_run_properties: bool = True + run_plan: RunPlan, + max_offers: Optional[int] = None, + include_run_properties: bool = True, + no_fleets: bool = False, ): run_spec = run_plan.get_effective_run_spec() job_plan = run_plan.job_plans[0] @@ -195,7 +203,7 @@ def th(s: str) -> str: ) console.print() else: - console.print(NO_OFFERS_WARNING) + console.print(NO_FLEETS_WARNING if no_fleets else NO_OFFERS_WARNING) def _format_run_status(run) -> str: @@ -215,8 +223,10 @@ def _format_run_status(run) -> str: RunStatus.FAILED: "indian_red1", RunStatus.DONE: "grey", } - if status_text == "no offers" or status_text == "interrupted": + if status_text in ("no offers", "interrupted"): color = "gold1" + elif status_text == "no fleets": + color = "indian_red1" elif status_text == "pulling": color = "sea_green3" else: @@ -230,6 +240,8 @@ def _format_job_submission_status(job_submission: JobSubmission, verbose: bool) job_status = job_submission.status if status_message in ("no offers", "interrupted"): color = "gold1" + elif status_message == "no fleets": + color = "indian_red1" elif status_message == "stopped": color = "grey" else: diff --git a/src/dstack/_internal/server/services/jobs/__init__.py b/src/dstack/_internal/server/services/jobs/__init__.py index 1ed3c5f99..68fea166c 100644 --- a/src/dstack/_internal/server/services/jobs/__init__.py +++ b/src/dstack/_internal/server/services/jobs/__init__.py @@ -804,6 +804,11 @@ def _get_job_status_message(job_model: JobModel) -> str: elif ( job_model.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY ): + if ( + job_model.termination_reason_message + and "No fleet found" in job_model.termination_reason_message + ): + return "no fleets" return "no offers" elif job_model.termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY: return "interrupted" diff --git a/src/tests/_internal/cli/utils/test_run.py b/src/tests/_internal/cli/utils/test_run.py index b824c001a..20f37a820 100644 --- a/src/tests/_internal/cli/utils/test_run.py +++ b/src/tests/_internal/cli/utils/test_run.py @@ -96,6 +96,7 @@ async def create_run_with_job( job_provisioning_data: Optional[JobProvisioningData] = None, termination_reason: Optional[JobTerminationReason] = None, exit_status: Optional[int] = None, + termination_reason_message: Optional[str] = None, submitted_at: Optional[datetime] = None, ) -> Run: if submitted_at is None: @@ -178,6 +179,9 @@ async def create_run_with_job( if exit_status is not None: job_model.exit_status = exit_status + if termination_reason_message is not None: + job_model.termination_reason_message = termination_reason_message + if exit_status is not None or termination_reason_message is not None: await session.commit() await session.refresh(run_model_db) @@ -226,13 +230,14 @@ async def test_simple_run(self, session: AsyncSession): assert status_style == "bold sea_green3" @pytest.mark.parametrize( - "job_status,termination_reason,exit_status,expected_status,expected_style", + "job_status,termination_reason,exit_status,termination_reason_message,expected_status,expected_style", [ - (JobStatus.DONE, None, None, "exited (0)", "grey"), + (JobStatus.DONE, None, None, None, "exited (0)", "grey"), ( JobStatus.FAILED, JobTerminationReason.CONTAINER_EXITED_WITH_ERROR, 1, + None, "exited (1)", "indian_red1", ), @@ -240,6 +245,7 @@ async def test_simple_run(self, session: AsyncSession): JobStatus.FAILED, JobTerminationReason.CONTAINER_EXITED_WITH_ERROR, 42, + None, "exited (42)", "indian_red1", ), @@ -247,13 +253,23 @@ async def test_simple_run(self, session: AsyncSession): JobStatus.FAILED, JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, None, + None, "no offers", "gold1", ), + ( + JobStatus.FAILED, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + None, + "No fleet found. Create it before submitting a run: https://dstack.ai/docs/concepts/fleets", + "no fleets", + "indian_red1", + ), ( JobStatus.FAILED, JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY, None, + None, "interrupted", "gold1", ), @@ -261,6 +277,7 @@ async def test_simple_run(self, session: AsyncSession): JobStatus.FAILED, JobTerminationReason.INSTANCE_UNREACHABLE, None, + None, "error", "indian_red1", ), @@ -268,14 +285,22 @@ async def test_simple_run(self, session: AsyncSession): JobStatus.TERMINATED, JobTerminationReason.TERMINATED_BY_USER, None, + None, "stopped", "grey", ), - (JobStatus.TERMINATED, JobTerminationReason.ABORTED_BY_USER, None, "aborted", "grey"), - (JobStatus.RUNNING, None, None, "running", "bold sea_green3"), - (JobStatus.PROVISIONING, None, None, "provisioning", "bold deep_sky_blue1"), - (JobStatus.PULLING, None, None, "pulling", "bold sea_green3"), - (JobStatus.TERMINATING, None, None, "terminating", "bold deep_sky_blue1"), + ( + JobStatus.TERMINATED, + JobTerminationReason.ABORTED_BY_USER, + None, + None, + "aborted", + "grey", + ), + (JobStatus.RUNNING, None, None, None, "running", "bold sea_green3"), + (JobStatus.PROVISIONING, None, None, None, "provisioning", "bold deep_sky_blue1"), + (JobStatus.PULLING, None, None, None, "pulling", "bold sea_green3"), + (JobStatus.TERMINATING, None, None, None, "terminating", "bold deep_sky_blue1"), ], ) async def test_status_messages( @@ -284,6 +309,7 @@ async def test_status_messages( job_status: JobStatus, termination_reason: Optional[JobTerminationReason], exit_status: Optional[int], + termination_reason_message: Optional[str], expected_status: str, expected_style: str, ): @@ -292,6 +318,7 @@ async def test_status_messages( job_status=job_status, termination_reason=termination_reason, exit_status=exit_status, + termination_reason_message=termination_reason_message, ) table = get_runs_table([api_run], verbose=False)