Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions frontend/src/libs/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,11 @@ export const getStatusIconType = (
export const getStatusIconColor = (
status: IRun['status'] | TJobStatus,
terminationReason: string | null | undefined,
statusMessage: string,
): StatusIndicatorProps.Color | undefined => {
if (statusMessage === 'No fleets') {
return 'red';
}
if (terminationReason === 'failed_to_start_due_to_no_capacity' || terminationReason === 'interrupted_by_no_capacity') {
return 'yellow';
}
Expand Down
6 changes: 4 additions & 2 deletions frontend/src/pages/Runs/Details/RunDetails/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ export const RunDetails = () => {

const finishedAt = getRunListFinishedAt(runData);

const statusMessage = getRunStatusMessage(runData);

return (
<>
<Container header={<Header variant="h2">{t('common.general')}</Header>}>
Expand Down Expand Up @@ -112,9 +114,9 @@ export const RunDetails = () => {
<div>
<StatusIndicator
type={getStatusIconType(status, terminationReason)}
colorOverride={getStatusIconColor(status, terminationReason)}
colorOverride={getStatusIconColor(status, terminationReason, statusMessage)}
>
{getRunStatusMessage(runData)}
{statusMessage}
</StatusIndicator>
</div>
</div>
Expand Down
5 changes: 3 additions & 2 deletions frontend/src/pages/Runs/List/hooks/useColumnsDefinitions.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,14 @@ export const useColumnsDefinitions = () => {
const terminationReason = finishedRunStatuses.includes(item.status)
? item.latest_job_submission?.termination_reason
: null;
const statusMessage = getRunStatusMessage(item);

return (
<StatusIndicator
type={getStatusIconType(status, terminationReason)}
colorOverride={getStatusIconColor(status, terminationReason)}
colorOverride={getStatusIconColor(status, terminationReason, statusMessage)}
>
{getRunStatusMessage(item)}
{statusMessage}
</StatusIndicator>
);
},
Expand Down
7 changes: 6 additions & 1 deletion src/dstack/_internal/cli/services/configurators/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,12 @@ def apply_configuration(
ssh_identity_file=configurator_args.ssh_identity_file,
)

print_run_plan(run_plan, max_offers=configurator_args.max_offers)
no_fleets = False
if len(run_plan.job_plans[0].offers) == 0:
if len(self.api.client.fleets.list(self.api.project)) == 0:
no_fleets = True

print_run_plan(run_plan, max_offers=configurator_args.max_offers, no_fleets=no_fleets)

confirm_message = "Submit a new run?"
if conf.name:
Expand Down
6 changes: 6 additions & 0 deletions src/dstack/_internal/cli/utils/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@
" https://dstack.ai/docs/guides/troubleshooting/#no-offers"
"[/]\n"
)
NO_FLEETS_WARNING = (
"[warning]"
"The project has no fleets. Create one before submitting a run:"
" https://dstack.ai/docs/concepts/fleets"
"[/]\n"
)


def cli_error(e: DstackError) -> CLIError:
Expand Down
20 changes: 16 additions & 4 deletions src/dstack/_internal/cli/utils/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@

from dstack._internal.cli.models.offers import OfferCommandOutput, OfferRequirements
from dstack._internal.cli.models.runs import PsCommandOutput
from dstack._internal.cli.utils.common import NO_OFFERS_WARNING, add_row_from_dict, console
from dstack._internal.cli.utils.common import (
NO_FLEETS_WARNING,
NO_OFFERS_WARNING,
add_row_from_dict,
console,
)
from dstack._internal.core.models.backends.base import BackendType
from dstack._internal.core.models.configurations import DevEnvironmentConfiguration
from dstack._internal.core.models.instances import (
Expand Down Expand Up @@ -75,7 +80,10 @@ def print_runs_json(project: str, runs: List[Run]) -> None:


def print_run_plan(
run_plan: RunPlan, max_offers: Optional[int] = None, include_run_properties: bool = True
run_plan: RunPlan,
max_offers: Optional[int] = None,
include_run_properties: bool = True,
no_fleets: bool = False,
):
run_spec = run_plan.get_effective_run_spec()
job_plan = run_plan.job_plans[0]
Expand Down Expand Up @@ -195,7 +203,7 @@ def th(s: str) -> str:
)
console.print()
else:
console.print(NO_OFFERS_WARNING)
console.print(NO_FLEETS_WARNING if no_fleets else NO_OFFERS_WARNING)


def _format_run_status(run) -> str:
Expand All @@ -215,8 +223,10 @@ def _format_run_status(run) -> str:
RunStatus.FAILED: "indian_red1",
RunStatus.DONE: "grey",
}
if status_text == "no offers" or status_text == "interrupted":
if status_text in ("no offers", "interrupted"):
color = "gold1"
elif status_text == "no fleets":
color = "indian_red1"
elif status_text == "pulling":
color = "sea_green3"
else:
Expand All @@ -230,6 +240,8 @@ def _format_job_submission_status(job_submission: JobSubmission, verbose: bool)
job_status = job_submission.status
if status_message in ("no offers", "interrupted"):
color = "gold1"
elif status_message == "no fleets":
color = "indian_red1"
elif status_message == "stopped":
color = "grey"
else:
Expand Down
5 changes: 5 additions & 0 deletions src/dstack/_internal/server/services/jobs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -804,6 +804,11 @@ def _get_job_status_message(job_model: JobModel) -> str:
elif (
job_model.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY
):
if (
job_model.termination_reason_message
and "No fleet found" in job_model.termination_reason_message
):
return "no fleets"
return "no offers"
elif job_model.termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY:
return "interrupted"
Expand Down
41 changes: 34 additions & 7 deletions src/tests/_internal/cli/utils/test_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,7 @@ async def create_run_with_job(
job_provisioning_data: Optional[JobProvisioningData] = None,
termination_reason: Optional[JobTerminationReason] = None,
exit_status: Optional[int] = None,
termination_reason_message: Optional[str] = None,
submitted_at: Optional[datetime] = None,
) -> Run:
if submitted_at is None:
Expand Down Expand Up @@ -178,6 +179,9 @@ async def create_run_with_job(

if exit_status is not None:
job_model.exit_status = exit_status
if termination_reason_message is not None:
job_model.termination_reason_message = termination_reason_message
if exit_status is not None or termination_reason_message is not None:
await session.commit()

await session.refresh(run_model_db)
Expand Down Expand Up @@ -226,56 +230,77 @@ async def test_simple_run(self, session: AsyncSession):
assert status_style == "bold sea_green3"

@pytest.mark.parametrize(
"job_status,termination_reason,exit_status,expected_status,expected_style",
"job_status,termination_reason,exit_status,termination_reason_message,expected_status,expected_style",
[
(JobStatus.DONE, None, None, "exited (0)", "grey"),
(JobStatus.DONE, None, None, None, "exited (0)", "grey"),
(
JobStatus.FAILED,
JobTerminationReason.CONTAINER_EXITED_WITH_ERROR,
1,
None,
"exited (1)",
"indian_red1",
),
(
JobStatus.FAILED,
JobTerminationReason.CONTAINER_EXITED_WITH_ERROR,
42,
None,
"exited (42)",
"indian_red1",
),
(
JobStatus.FAILED,
JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY,
None,
None,
"no offers",
"gold1",
),
(
JobStatus.FAILED,
JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY,
None,
"No fleet found. Create it before submitting a run: https://dstack.ai/docs/concepts/fleets",
"no fleets",
"indian_red1",
),
(
JobStatus.FAILED,
JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY,
None,
None,
"interrupted",
"gold1",
),
(
JobStatus.FAILED,
JobTerminationReason.INSTANCE_UNREACHABLE,
None,
None,
"error",
"indian_red1",
),
(
JobStatus.TERMINATED,
JobTerminationReason.TERMINATED_BY_USER,
None,
None,
"stopped",
"grey",
),
(JobStatus.TERMINATED, JobTerminationReason.ABORTED_BY_USER, None, "aborted", "grey"),
(JobStatus.RUNNING, None, None, "running", "bold sea_green3"),
(JobStatus.PROVISIONING, None, None, "provisioning", "bold deep_sky_blue1"),
(JobStatus.PULLING, None, None, "pulling", "bold sea_green3"),
(JobStatus.TERMINATING, None, None, "terminating", "bold deep_sky_blue1"),
(
JobStatus.TERMINATED,
JobTerminationReason.ABORTED_BY_USER,
None,
None,
"aborted",
"grey",
),
(JobStatus.RUNNING, None, None, None, "running", "bold sea_green3"),
(JobStatus.PROVISIONING, None, None, None, "provisioning", "bold deep_sky_blue1"),
(JobStatus.PULLING, None, None, None, "pulling", "bold sea_green3"),
(JobStatus.TERMINATING, None, None, None, "terminating", "bold deep_sky_blue1"),
],
)
async def test_status_messages(
Expand All @@ -284,6 +309,7 @@ async def test_status_messages(
job_status: JobStatus,
termination_reason: Optional[JobTerminationReason],
exit_status: Optional[int],
termination_reason_message: Optional[str],
expected_status: str,
expected_style: str,
):
Expand All @@ -292,6 +318,7 @@ async def test_status_messages(
job_status=job_status,
termination_reason=termination_reason,
exit_status=exit_status,
termination_reason_message=termination_reason_message,
)

table = get_runs_table([api_run], verbose=False)
Expand Down