diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df5ecccf81..6877b1eb06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,8 @@ jobs: - Python 3.10 Tests - Python 3.11 Tests - Python 3.12 Tests + - Python 3.13 Tests + - Python 3.14 Tests - Python 3.10 Tests Coverage - Code Checks - CLI CloudFormation Templates Checks @@ -64,6 +66,14 @@ jobs: python: '3.12' toxdir: cli toxenv: py312-nocov + - name: Python 3.13 Tests + python: '3.13' + toxdir: cli + toxenv: py313-nocov + - name: Python 3.14 Tests + python: '3.14' + toxdir: cli + toxenv: py314-nocov - name: Python 3.10 Tests Coverage python: '3.10' toxdir: cli @@ -113,6 +123,8 @@ jobs: - Python 3.10 AWS Batch CLI Tests - Python 3.11 AWS Batch CLI Tests - Python 3.12 AWS Batch CLI Tests + - Python 3.13 AWS Batch CLI Tests + - Python 3.14 AWS Batch CLI Tests - Python 3.10 AWS Batch CLI Tests Coverage - Code Checks AWS Batch CLI include: @@ -132,6 +144,14 @@ jobs: python: '3.12' toxdir: awsbatch-cli toxenv: py312-nocov + - name: Python 3.13 AWS Batch CLI Tests + python: '3.13' + toxdir: awsbatch-cli + toxenv: py313-nocov + - name: Python 3.14 AWS Batch CLI Tests + python: '3.14' + toxdir: awsbatch-cli + toxenv: py314-nocov - name: Python 3.10 AWS Batch CLI Tests Coverage python: '3.10' toxdir: awsbatch-cli diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a01c9cb48..5a26c2d50b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ CHANGELOG - Add validator that warns against the downsides of disabling in-place updates on compute and login nodes through DevSettings. - Upgrade jmespath to ~=1.0 (from ~=0.10). - Upgrade tabulate to <=0.9.0 (from <=0.8.10). +- Add support for Python 3.14. **BUG FIXES** - Add validation to block updates that change tag order. Blocking such change prevents update failures. diff --git a/awsbatch-cli/src/awsbatch/utils.py b/awsbatch-cli/src/awsbatch/utils.py index f102695d22..9b45bebfc0 100644 --- a/awsbatch-cli/src/awsbatch/utils.py +++ b/awsbatch-cli/src/awsbatch/utils.py @@ -10,8 +10,8 @@ # This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. # See the License for the specific language governing permissions and limitations under the License. -import pipes import re +import shlex import sys from datetime import datetime from typing import NoReturn @@ -73,7 +73,7 @@ def shell_join(array): :param array: input array :return: the shell-quoted string """ - return " ".join(pipes.quote(item) for item in array) + return " ".join(shlex.quote(item) for item in array) def is_job_array(job): diff --git a/awsbatch-cli/tox.ini b/awsbatch-cli/tox.ini index 8667dbc0c7..a1f2fab5dd 100644 --- a/awsbatch-cli/tox.ini +++ b/awsbatch-cli/tox.ini @@ -1,7 +1,7 @@ [tox] toxworkdir=../.tox envlist = - py{37,38,39,310}-{cov,nocov} + py{39,310,311,312,313,314}-{cov,nocov} code-linters # Default testenv. Used to run tests on all python versions. diff --git a/cli/.flake8 b/cli/.flake8 index e8e5f41bbc..8286b2bd62 100644 --- a/cli/.flake8 +++ b/cli/.flake8 @@ -10,6 +10,8 @@ ignore = D107, # D103: Missing docstring in public function D103, + # D209: Multi-line docstring closing quotes should be on a separate line => Conflicts with black style. + D209, # W503: line break before binary operator => Conflicts with black style. W503, # D413: Missing blank line after last section diff --git a/cli/setup.py b/cli/setup.py index e3cd2ffc2e..84672fef11 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -69,7 +69,7 @@ def readme(): license="Apache License 2.0", package_dir={"": "src"}, packages=find_namespace_packages("src"), - python_requires=">=3.9", + python_requires=">=3.9, <3.15", install_requires=REQUIRES, extras_require={ "awslambda": LAMBDA_REQUIRES, @@ -92,6 +92,8 @@ def readme(): "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "License :: OSI Approved :: Apache Software License", ], diff --git a/cli/src/pcluster/cli/commands/configure/easyconfig.py b/cli/src/pcluster/cli/commands/configure/easyconfig.py index 62c4569d94..9283762815 100644 --- a/cli/src/pcluster/cli/commands/configure/easyconfig.py +++ b/cli/src/pcluster/cli/commands/configure/easyconfig.py @@ -177,9 +177,9 @@ def configure(args): # noqa: C901 for queue_index in range(number_of_queues): while True: queue_name = prompt( - f"Name of queue {queue_index+1}", + f"Name of queue {queue_index + 1}", validator=lambda x: len(NameValidator().execute(x)) == 0, - default_value=f"queue{queue_index+1}", + default_value=f"queue{queue_index + 1}", ) if queue_name not in queue_names: break @@ -208,7 +208,7 @@ def configure(args): # noqa: C901 if scheduler != "awsbatch": while True: compute_instance_type = prompt( - f"Compute instance type for compute resource {compute_resource_index+1} in {queue_name}", + f"Compute instance type for compute resource {compute_resource_index + 1} in {queue_name}", validator=lambda x: x in AWSApi.instance().ec2.list_instance_types(), default_value=default_instance_type, ) diff --git a/cli/src/pcluster/config/common.py b/cli/src/pcluster/config/common.py index 4633e1bdcb..a19baee1fc 100644 --- a/cli/src/pcluster/config/common.py +++ b/cli/src/pcluster/config/common.py @@ -20,6 +20,7 @@ from enum import Enum from typing import List, Set +from pcluster.utils import get_or_create_event_loop from pcluster.validators.common import AsyncValidator, FailureLevel, ValidationResult, Validator, ValidatorContext from pcluster.validators.dev_settings_validators import ExtraChefAttributesValidator from pcluster.validators.iam_validators import AdditionalIamPolicyValidator @@ -210,7 +211,7 @@ def _await_async_validators(self): # does not cascade to child resources return list( itertools.chain.from_iterable( - asyncio.get_event_loop().run_until_complete(asyncio.gather(*self._validation_futures)) + get_or_create_event_loop().run_until_complete(asyncio.gather(*self._validation_futures)) ) ) diff --git a/cli/src/pcluster/config/config_patch.py b/cli/src/pcluster/config/config_patch.py index a2297f882d..ebfe0c8071 100644 --- a/cli/src/pcluster/config/config_patch.py +++ b/cli/src/pcluster/config/config_patch.py @@ -221,11 +221,7 @@ def _compare_list(self, base_section, target_section, param_path, data_key, fiel for target_nested_section in target_list: update_key_value = target_nested_section.get(update_key) base_nested_section = next( - ( - nested_section - for nested_section in base_list - if nested_section.get(update_key) == update_key_value - ), + (nested_section for nested_section in base_list if nested_section.get(update_key) == update_key_value), None, ) if base_nested_section: diff --git a/cli/src/pcluster/resources/custom_resources/custom_resources_code/crhelper/resource_helper.py b/cli/src/pcluster/resources/custom_resources/custom_resources_code/crhelper/resource_helper.py index cc1960a7d5..86a6e58bff 100644 --- a/cli/src/pcluster/resources/custom_resources/custom_resources_code/crhelper/resource_helper.py +++ b/cli/src/pcluster/resources/custom_resources/custom_resources_code/crhelper/resource_helper.py @@ -39,7 +39,7 @@ def __init__( polling_interval=2, sleep_on_delete=120 ): - self._sleep_on_delete= sleep_on_delete + self._sleep_on_delete = sleep_on_delete self._create_func = None self._update_func = None self._delete_func = None @@ -93,7 +93,7 @@ def __call__(self, event, context): else: logger.debug("enabling send_response") self._send_response = True - logger.debug("_send_response: %s", self._send_response) + logger.debug("_send_response: %s", self._send_response) if self._send_response: if self.RequestType == 'Delete': self._wait_for_cwlogs() diff --git a/cli/src/pcluster/utils.py b/cli/src/pcluster/utils.py index ac3f4c36e9..681d56e36d 100644 --- a/cli/src/pcluster/utils.py +++ b/cli/src/pcluster/utils.py @@ -483,6 +483,18 @@ def batch_by_property_callback(items, property_callback: Callable[..., int], bat yield current_batch +def get_or_create_event_loop(): + """Get the current event loop or create a new one. + + This approach is required to support Python 3.14 and maintain retrocompatibility with Python 3.8+.""" + try: + return asyncio.get_running_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + return loop + + class AsyncUtils: """Utility class for async functions.""" @@ -559,7 +571,7 @@ def async_from_sync(func): @functools.wraps(func) async def wrapper(self, *args, **kwargs): - return await asyncio.get_event_loop().run_in_executor( + return await get_or_create_event_loop().run_in_executor( AsyncUtils._thread_pool_executor, lambda: func(self, *args, **kwargs) ) diff --git a/cli/src/pcluster/validators/common.py b/cli/src/pcluster/validators/common.py index a6aa04759d..c2fdae2d70 100644 --- a/cli/src/pcluster/validators/common.py +++ b/cli/src/pcluster/validators/common.py @@ -20,6 +20,7 @@ from typing import List from pcluster.aws.common import AWSClientError +from pcluster.utils import get_or_create_event_loop ASYNC_TIMED_VALIDATORS_DEFAULT_TIMEOUT_SEC = 10 @@ -84,7 +85,7 @@ def __init__(self): super().__init__() def _validate(self, *arg, **kwargs): - asyncio.get_event_loop().run_until_complete(self._validate_async(*arg, **kwargs)) + get_or_create_event_loop().run_until_complete(self._validate_async(*arg, **kwargs)) return self._failures async def execute_async(self, *arg, **kwargs) -> List[ValidationResult]: diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py index 877af93d09..2a91bd1612 100644 --- a/cli/tests/conftest.py +++ b/cli/tests/conftest.py @@ -237,14 +237,46 @@ def _run_cli(command, expect_failure=False, expect_message=None): return _run_cli +def normalize_argparse_help_text(text): + """ + Normalize argparse help text to be version-agnostic for comparison. + + Python version differences in argparse help output: + - Python 3.10+: Changed 'optional arguments:' to 'options:' + - Python 3.13+: Changed '-r REGION, --region REGION' to '-r, --region REGION' + (the metavar is no longer repeated for the short option) + - Python 3.13+: Changed help text alignment/wrapping + + This function normalizes text to enable comparison across Python versions. + """ + import re + + # Normalize 'optional arguments:' vs 'options:' (Python 3.10 change) + text = text.replace("optional arguments:", "options:") + + # Normalize short option format: '-X METAVAR, --long' -> '-X, --long' + # This handles the Python 3.13 change where metavar is no longer repeated + text = re.sub(r"(-[a-zA-Z]) ([A-Z][A-Z_]*), (--[a-z][a-z-]*) \2", r"\1, \3 \2", text) + + # Normalize whitespace: collapse multiple spaces/newlines into single space + # This handles alignment differences across Python versions + text = re.sub(r"\s+", " ", text) + + return text + + @pytest.fixture() def assert_out_err(capsys): def _assert_out_err(expected_out, expected_err): out_err = capsys.readouterr() - # In Python 3.10 ArgParse renamed the 'optional arguments' section in the helper to 'option' - expected_out_alternative = expected_out.replace("options", "optional arguments") + actual_out = out_err.out.strip() + + # Normalize both expected and actual output for version-agnostic comparison + normalized_expected = normalize_argparse_help_text(expected_out) + normalized_actual = normalize_argparse_help_text(actual_out) + with soft_assertions(): - assert_that(out_err.out.strip()).is_in(expected_out, expected_out_alternative) + assert_that(normalized_actual).is_equal_to(normalized_expected) assert_that(out_err.err.strip()).contains(expected_err) return _assert_out_err diff --git a/cli/tests/pcluster/test_utils.py b/cli/tests/pcluster/test_utils.py index 6e2df607fc..c6ec2c1933 100644 --- a/cli/tests/pcluster/test_utils.py +++ b/cli/tests/pcluster/test_utils.py @@ -565,7 +565,7 @@ async def async_method(self, param): executions.append(FakeAsyncMethodProvider().async_method(i)) expected_results.append(i) - results = asyncio.get_event_loop().run_until_complete(asyncio.gather(*executions)) + results = utils.get_or_create_event_loop().run_until_complete(asyncio.gather(*executions)) assert_that(expected_results).contains_sequence(*results) assert_that(unique_calls).is_equal_to(total_calls) diff --git a/cli/tox.ini b/cli/tox.ini index 1d1c6206b3..71478c519c 100644 --- a/cli/tox.ini +++ b/cli/tox.ini @@ -1,7 +1,7 @@ [tox] toxworkdir=../.tox envlist = - py{39,310,311,312}-{cov,nocov} + py{39,310,311,312,313,314}-{cov,nocov} code-linters cfn-{tests,lint,format-check} @@ -82,14 +82,48 @@ skip_install = true deps = {[testenv:isort]deps} commands = {[testenv:isort]commands} --check --diff -# Reformats code with black and isort. +# autopep8 fixes PEP8 issues that black doesn't handle (E201, E202, E226, etc.) +# https://github.com/hhatto/autopep8 +[testenv:autopep8] +basepython = python3 +skip_install = true +deps = + autopep8 +commands = + autopep8 --in-place --recursive --max-line-length 120 \ + --select=E201,E202,E203,E211,E225,E226,E227,E228,E231,E241,E242,E251,E252,E261,E262,E265,E266,E271,E272,E273,E274,E275 \ + {[vars]code_dirs} \ + {posargs} + +# docformatter fixes docstring formatting issues (D209, D400, etc.) +# https://github.com/PyCQA/docformatter +# Note: --close-quotes-on-newline fixes D209 (closing quotes on separate line) +; [testenv:docformatter] +; basepython = python3 +; skip_install = true +; deps = +; docformatter +; commands = +; docformatter --in-place --recursive --wrap-summaries 120 --wrap-descriptions 120 \ +; --close-quotes-on-newline \ +; {[vars]code_dirs} \ +; {posargs} + +# Reformats code with autopep8, docformatter, isort, and black. +# Order matters: autopep8 first (fixes spacing), docformatter (fixes docstrings), +# then isort (sorts imports), then black (final formatting) +# Note: docformatter returns exit code 3 when it makes changes, so we use '-' to ignore it [testenv:autoformat] basepython = python3 skip_install = true deps = + {[testenv:autopep8]deps} + ; {[testenv:docformatter]deps} {[testenv:isort]deps} {[testenv:black]deps} commands = + {[testenv:autopep8]commands} + ; - {[testenv:docformatter]commands} {[testenv:isort]commands} {[testenv:black]commands} diff --git a/tests/integration-tests/tests/create/test_create.py b/tests/integration-tests/tests/create/test_create.py index 57257d6545..9109bb3e8c 100644 --- a/tests/integration-tests/tests/create/test_create.py +++ b/tests/integration-tests/tests/create/test_create.py @@ -159,7 +159,7 @@ def test_cluster_creation_with_problematic_preinstall_script( assert_lines_in_logs( remote_command_executor, ["/var/log/cfn-init.log"], - [f"Failed to execute OnNodeStart script 1 s3://{ bucket_name }/scripts/{script_name}"], + [f"Failed to execute OnNodeStart script 1 s3://{bucket_name}/scripts/{script_name}"], ) logging.info("Verifying error in cloudformation failure reason") stack_events = cluster.get_stack_events().get("events") @@ -173,7 +173,7 @@ def test_cluster_creation_with_problematic_preinstall_script( ) assert_that(cfn_failure_reason).contains(expected_cfn_failure_reason) - assert_that(cfn_failure_reason).does_not_contain(f"s3://{ bucket_name }/scripts/{script_name}") + assert_that(cfn_failure_reason).does_not_contain(f"s3://{bucket_name}/scripts/{script_name}") logging.info("Verifying failures in describe-clusters output") expected_failures = [ diff --git a/tests/integration-tests/tests/schedulers/test_slurm.py b/tests/integration-tests/tests/schedulers/test_slurm.py index 483ec9d8bb..66df4a5124 100644 --- a/tests/integration-tests/tests/schedulers/test_slurm.py +++ b/tests/integration-tests/tests/schedulers/test_slurm.py @@ -169,7 +169,7 @@ def test_slurm_ticket_17399( "partition": "gpu", "test_only": True, "other_options": f"--gpus {gpus_per_instance} --nodes 1 --ntasks-per-node 1 " - f"--cpus-per-task={cpus_per_instance//gpus_per_instance}", + f"--cpus-per-task={cpus_per_instance // gpus_per_instance}", } ) @@ -180,7 +180,7 @@ def test_slurm_ticket_17399( "partition": "gpu", "test_only": True, "other_options": f"--gpus {gpus_per_instance} --nodes 1 --ntasks-per-node 1 " - f"--cpus-per-task={cpus_per_instance//gpus_per_instance + 1}", + f"--cpus-per-task={cpus_per_instance // gpus_per_instance + 1}", } ) diff --git a/tests/integration-tests/tests/storage/test_ebs.py b/tests/integration-tests/tests/storage/test_ebs.py index 4311b4bf8a..94912563a9 100644 --- a/tests/integration-tests/tests/storage/test_ebs.py +++ b/tests/integration-tests/tests/storage/test_ebs.py @@ -182,7 +182,7 @@ def test_ebs_multiple( for i in range(len(volume_ids)): # test different volume types volume_id = volume_ids[i] - ebs_settings = _get_ebs_settings_by_name(cluster.config, f"ebs{i+1}") + ebs_settings = _get_ebs_settings_by_name(cluster.config, f"ebs{i + 1}") volume_type = ebs_settings["VolumeType"] volume = describe_volume(volume_id, region) assert_that(volume[0]).is_equal_to(volume_type)