From 3440ab9b936db249f9ef748494dd429f3d2a7e9e Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 3 Sep 2021 11:01:40 -0500 Subject: [PATCH 01/70] Update ci.yml --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd940825..6e776334 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,12 @@ jobs: python -m pip install --upgrade pip chmod +x ci_deploy.sh ./ci_deploy.sh - - name: Test with unittest + - name: Test unit tests with unittest run: | python -m unittest - + - name: Test integration tests for shells/generic_terraform_service with unittest + run: | + pwd + cd shells/generic_terraform_service + pwd + run the "python -m unittest discover -p "int_test_mock*.py" From cfb28d8e7b3d8ec273eae564bdef3e7a2e71bfe8 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 3 Sep 2021 11:03:26 -0500 Subject: [PATCH 02/70] Update ci.yml --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e776334..583d9b3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,8 +48,8 @@ jobs: run: | python -m unittest - name: Test integration tests for shells/generic_terraform_service with unittest - run: | - pwd - cd shells/generic_terraform_service - pwd - run the "python -m unittest discover -p "int_test_mock*.py" + run: | + pwd + cd shells/generic_terraform_service + pwd + run the "python -m unittest discover -p "int_test_mock*.py" From 455a702e94e067e8629c4f4675ca63f36f850776 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 3 Sep 2021 11:05:03 -0500 Subject: [PATCH 03/70] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 583d9b3b..33a3179a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,4 +52,4 @@ jobs: pwd cd shells/generic_terraform_service pwd - run the "python -m unittest discover -p "int_test_mock*.py" + python -m unittest discover -p "int_test_mock*.py" From 742d311c87ee664d068f5bd8633f4d2a083abfe7 Mon Sep 17 00:00:00 2001 From: qualidan Date: Tue, 7 Sep 2021 15:14:01 -0500 Subject: [PATCH 04/70] fixed type list to List --- package/requirements.txt | 2 +- .../helper_services/service_attributes_factory.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/package/requirements.txt b/package/requirements.txt index af1af792..7a534a7f 100644 --- a/package/requirements.txt +++ b/package/requirements.txt @@ -1,5 +1,5 @@ cloudshell-shell-core==5.0.5 requests==2.25.1 -cloudshell-automation-api==2021.1.0.181140 +cloudshell-automation-api==2021.2.0.182230 retry==0.9.2 python-hcl2==2.0.1 \ No newline at end of file diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_services/service_attributes_factory.py b/shells/generic_terraform_service/tests/integration_tests/helper_services/service_attributes_factory.py index 1803bc72..f893e71c 100644 --- a/shells/generic_terraform_service/tests/integration_tests/helper_services/service_attributes_factory.py +++ b/shells/generic_terraform_service/tests/integration_tests/helper_services/service_attributes_factory.py @@ -1,3 +1,5 @@ +from typing import List + from cloudshell.api.cloudshell_api import NameValuePair from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES @@ -5,7 +7,7 @@ class ServiceAttributesFactory: @staticmethod - def create_empty_attributes() -> list[dict]: + def create_empty_attributes() -> List[dict]: attributes = [ NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", Value=""), NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.BRANCH}", Value=""), From a4288cf30573c521af70814891d4eacd422e2d32 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 8 Sep 2021 13:58:07 -0500 Subject: [PATCH 05/70] commit before changing the way the attribute values are set during the test --- .../tests/integration_tests/.env.int_tests | 2 ++ .../helper_objects/env_vars.py | 3 +-- .../helper_objects/integration_context.py | 21 ++++++++++++------- .../helper_objects/integration_helper.py | 6 +++--- .../int_test_mock_tf_execute_destroy.py | 4 +++- .../tests/test_requirements.txt | 3 ++- 6 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 shells/generic_terraform_service/tests/integration_tests/.env.int_tests diff --git a/shells/generic_terraform_service/tests/integration_tests/.env.int_tests b/shells/generic_terraform_service/tests/integration_tests/.env.int_tests new file mode 100644 index 00000000..570dec65 --- /dev/null +++ b/shells/generic_terraform_service/tests/integration_tests/.env.int_tests @@ -0,0 +1,2 @@ +SB_SERVICE_ALIAS1 = "ooo1" +SB_SERVICE_ALIAS2 = "ooo2" diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_objects/env_vars.py b/shells/generic_terraform_service/tests/integration_tests/helper_objects/env_vars.py index 80d458c7..5f99f198 100644 --- a/shells/generic_terraform_service/tests/integration_tests/helper_objects/env_vars.py +++ b/shells/generic_terraform_service/tests/integration_tests/helper_objects/env_vars.py @@ -4,11 +4,10 @@ class EnvVars(object): - def __init__(self, service_name: str): + def __init__(self): load_dotenv() self.cs_user = os.environ.get("CS_USERNAME") self.cs_pass = os.environ.get("CS_PASSWORD") self.cs_server = os.environ.get("CS_SERVER") self.cs_domain = os.environ.get("RESERVATION_DOMAIN") self.cs_res_id = os.environ.get("RESERVATION_ID") - self.sb_service_alias = service_name diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py b/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py index 3528c41f..6da43cbe 100644 --- a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py +++ b/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py @@ -12,9 +12,11 @@ class IntegrationData(object): def __init__(self, service_name: str, is_api_real: bool = True, mock_api: Mock = None): - self._env_vars = EnvVars(service_name) + # + self._sb_service_alias = service_name if is_api_real: + self._env_vars = EnvVars() self.api = CloudShellAPISession( self._env_vars.cs_server, self._env_vars.cs_user, @@ -32,26 +34,29 @@ def __init__(self, service_name: str, is_api_real: bool = True, mock_api: Mock = def _set_context(self, is_api_real: bool): self.context = mock.create_autospec(ResourceCommandContext) self.context.connectivity = mock.MagicMock() - self.context.connectivity.server_address = self._env_vars.cs_server - self.context.connectivity.admin_auth_token = self.api.authentication.xmlrpc_token + if is_api_real: + self.context.connectivity.server_address = self._env_vars.cs_server + self.context.connectivity.admin_auth_token = self.api.authentication.xmlrpc_token self.context.resource = mock.MagicMock() self.context.resource.attributes = dict() - self.context.resource.name = self._env_vars.sb_service_alias + self.context.resource.name = self._sb_service_alias self.context.resource.model = 'Generic Terraform Service' if is_api_real: self.set_context_resource_attributes_from_cs() + self.context.reservation = mock.MagicMock() + self.context.reservation.reservation_id = self._env_vars.cs_res_id + self.context.reservation.domain = self._env_vars.cs_domain else: self.context.resource.attributes = ServiceAttributesFactory.create_empty_attributes() - self.context.reservation = mock.MagicMock() - self.context.reservation.reservation_id = self._env_vars.cs_res_id - self.context.reservation.domain = self._env_vars.cs_domain + self.context.reservation = mock.MagicMock() + def set_context_resource_attributes_from_cs(self, the_only_attribute_to_update: str = ""): services = self.api.GetReservationDetails(self._env_vars.cs_res_id, disableCache=True) \ .ReservationDescription.Services for service in services: - if service.Alias == self._env_vars.sb_service_alias: + if service.Alias == self._sb_service_alias: for attribute in service.Attributes: if the_only_attribute_to_update and attribute.Name == the_only_attribute_to_update: self.context.resource.attributes[attribute.Name] = attribute.Value diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py b/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py index 90a3ce5f..a7c92d51 100644 --- a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py +++ b/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py @@ -41,7 +41,7 @@ def wipe_uuids(sb: Sandbox): sb = Sandbox() -print_uuids(sb) -wipe_uuids(sb) -data = sb.automation_api.GetSandboxData(sb_id) +#print_uuids(sb) +#wipe_uuids(sb) +#data = sb.automation_api.GetSandboxData(sb_id) print("") diff --git a/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 27c012c3..83fffc0c 100644 --- a/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -10,13 +10,15 @@ import os from unittest import TestCase, mock +from pathlib import Path + from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesFactory class TestMockTerraformExecuteDestroy(TestCase): @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') def setUp(self, patched_api) -> None: - load_dotenv() + load_dotenv(Path('.env.int_tests')) self._prepare_mock_api() patched_api.return_value.get_api.return_value = self.mock_api diff --git a/shells/generic_terraform_service/tests/test_requirements.txt b/shells/generic_terraform_service/tests/test_requirements.txt index d8860af1..248fa44a 100644 --- a/shells/generic_terraform_service/tests/test_requirements.txt +++ b/shells/generic_terraform_service/tests/test_requirements.txt @@ -1,2 +1,3 @@ cloudshell-iac-terraform==0.1.0 -python-dotenv==0.18.0 \ No newline at end of file +python-dotenv==0.18.0 +pathlib==1.0.1 \ No newline at end of file From ae8a386c1a78c291186e0e1285212434529a395b Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 16 Sep 2021 11:18:21 -0500 Subject: [PATCH 06/70] Update integration_context.py --- .../helper_objects/integration_context.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index 7cf6e317..59876326 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -10,12 +10,14 @@ class IntegrationData(object): def __init__(self): + self._env_vars = EnvVars() self.real_api = CloudShellAPISession( self._env_vars.cs_server, self._env_vars.cs_user, self._env_vars.cs_pass, - self._env_vars.cs_domain + self._env_vars.cs_domain, + os.environ.get("SB_SERVICE_ALIAS") ) self._set_context() self._logger = get_qs_logger(log_group=self.context.resource.name) @@ -44,8 +46,3 @@ def set_context_resource_attributes(self): if service.Alias == self._env_vars.sb_service_alias: for attribute in service.Attributes: self.context.resource.attributes[attribute.Name] = attribute.Value - ''' - def _create_driver(self) : - self.driver = GenericTerraformServiceDriver() - self.driver.initialize(self.context) - ''' From b29c9a49152ae4e39bc960675364eca306fb2e64 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 24 Sep 2021 16:13:43 -0500 Subject: [PATCH 07/70] relocated tests --- package/test_requirements.txt | 1 + package/tests/integration_tests/constants.py | 19 ++ .../downloader_integration_context.py | 49 ++++ .../helper_objects/integration_context.py | 75 +++--- .../service_attributes_factory.py | 25 ++ .../integration_tests/int_test_downloader.py | 2 +- .../int_test_mock_tf_execute_destroy.py | 226 ++++++++++++++++++ .../int_test_real_tf_execute_destroy.py | 209 ++++++++++++++++ package/tests/integration_tests/int_tests.env | 12 + .../integration_tests/int_tests.template.env | 13 + .../integration_tests/int_test_validate.py | 2 +- .../helper_objects/integration_context.py | 1 - .../int_test_mock_tf_execute_destroy.py | 2 +- .../int_test_real_tf_execute_destroy.py | 2 +- .../{.env.int_tests => int_tests.env} | 0 15 files changed, 606 insertions(+), 32 deletions(-) create mode 100644 package/tests/integration_tests/constants.py create mode 100644 package/tests/integration_tests/helper_objects/downloader_integration_context.py create mode 100644 package/tests/integration_tests/helper_services/service_attributes_factory.py create mode 100644 package/tests/integration_tests/int_test_mock_tf_execute_destroy.py create mode 100644 package/tests/integration_tests/int_test_real_tf_execute_destroy.py create mode 100644 package/tests/integration_tests/int_tests.env create mode 100644 package/tests/integration_tests/int_tests.template.env rename shells/generic_terraform_service/tests/integration_tests/{.env.int_tests => int_tests.env} (100%) diff --git a/package/test_requirements.txt b/package/test_requirements.txt index 4f700872..dace1ea8 100644 --- a/package/test_requirements.txt +++ b/package/test_requirements.txt @@ -1,2 +1,3 @@ mock==4.0.3 python-dotenv==0.18.0 +pathlib==1.0.1 diff --git a/package/tests/integration_tests/constants.py b/package/tests/integration_tests/constants.py new file mode 100644 index 00000000..36e27876 --- /dev/null +++ b/package/tests/integration_tests/constants.py @@ -0,0 +1,19 @@ +SHELL_NAME = "Generic Terraform Service" +UUID_ATTRIBUTE = f"{SHELL_NAME}.UUID" + + +class ATTRIBUTE_NAMES: + TF_OUTPUTS = "Terraform Outputs" + TF_SENSIITVE_OUTPUTS = "Terraform Sensitive Outputs" + TF_INPUTS = "Terraform Inputs" + CT_INPUTS = "Custom Tags" + APPLY_TAGS = "Apply Tags" + CUSTOM_TAGS = "Custom Tags" + REMOTE_STATE_PROVIDER = "Remote State Provider" + GITHUB_TERRAFORM_MODULE_URL = "Github Terraform Module URL" + TERRAFORM_VERSION = "Terraform Version" + GITHUB_TOKEN = "Github Token" + GITHUB_URL = "Github Terraform Module URL" + BRANCH = "Branch" + CLOUD_PROVIDER = "Cloud Provider" + UUID = "UUID" diff --git a/package/tests/integration_tests/helper_objects/downloader_integration_context.py b/package/tests/integration_tests/helper_objects/downloader_integration_context.py new file mode 100644 index 00000000..6620ea4c --- /dev/null +++ b/package/tests/integration_tests/helper_objects/downloader_integration_context.py @@ -0,0 +1,49 @@ +from unittest import mock +import os + +from cloudshell.api.cloudshell_api import CloudShellAPISession +from cloudshell.logging.qs_logger import get_qs_logger +from cloudshell.shell.core.driver_context import ResourceCommandContext + + +from package.tests.integration_tests.helper_objects.env_vars import EnvVars + + +class IntegrationData(object): + def __init__(self): + + self._env_vars = EnvVars() + self.real_api = CloudShellAPISession( + self._env_vars.cs_server, + self._env_vars.cs_user, + self._env_vars.cs_pass, + self._env_vars.cs_domain, + os.environ.get("SB_SERVICE_ALIAS") + ) + self._set_context() + self._logger = get_qs_logger(log_group=self.context.resource.name) + + # self._create_driver() + + def _set_context(self): + self.context = mock.create_autospec(ResourceCommandContext) + self.context.connectivity = mock.MagicMock() + self.context.connectivity.server_address = self._env_vars.cs_server + self.context.connectivity.admin_auth_token = self.real_api.authentication.xmlrpc_token + + self.context.resource = mock.MagicMock() + self.context.resource.attributes = dict() + self.context.resource.name = self._env_vars.sb_service_alias + self.context.resource.model = 'Generic Terraform Service' + self.set_context_resource_attributes() + + self.context.reservation = mock.MagicMock() + self.context.reservation.reservation_id = self._env_vars.cs_res_id + self.context.reservation.domain = self._env_vars.cs_domain + + def set_context_resource_attributes(self): + services = self.real_api.GetReservationDetails(self._env_vars.cs_res_id).ReservationDescription.Services + for service in services: + if service.Alias == self._env_vars.sb_service_alias: + for attribute in service.Attributes: + self.context.resource.attributes[attribute.Name] = attribute.Value diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index 59876326..6da43cbe 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -1,48 +1,69 @@ from unittest import mock +from unittest.mock import Mock from cloudshell.api.cloudshell_api import CloudShellAPISession +from cloudshell.iac.terraform import TerraformShell, TerraformShellConfig from cloudshell.logging.qs_logger import get_qs_logger from cloudshell.shell.core.driver_context import ResourceCommandContext - -from package.tests.integration_tests.helper_objects.env_vars import EnvVars +from tests.integration_tests.helper_objects.env_vars import EnvVars +from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesFactory class IntegrationData(object): - def __init__(self): - - self._env_vars = EnvVars() - self.real_api = CloudShellAPISession( - self._env_vars.cs_server, - self._env_vars.cs_user, - self._env_vars.cs_pass, - self._env_vars.cs_domain, - os.environ.get("SB_SERVICE_ALIAS") - ) - self._set_context() - self._logger = get_qs_logger(log_group=self.context.resource.name) + def __init__(self, service_name: str, is_api_real: bool = True, mock_api: Mock = None): + # + self._sb_service_alias = service_name - # self._create_driver() + if is_api_real: + self._env_vars = EnvVars() + self.api = CloudShellAPISession( + self._env_vars.cs_server, + self._env_vars.cs_user, + self._env_vars.cs_pass, + self._env_vars.cs_domain + ) - def _set_context(self): + else: + self.api = mock_api + self.api.authentication.xmlrpc_token = Mock() + self._set_context(is_api_real) + self._logger = get_qs_logger(log_group=self.context.resource.name) + self.create_tf_shell() + + def _set_context(self, is_api_real: bool): self.context = mock.create_autospec(ResourceCommandContext) self.context.connectivity = mock.MagicMock() - self.context.connectivity.server_address = self._env_vars.cs_server - self.context.connectivity.admin_auth_token = self.real_api.authentication.xmlrpc_token + if is_api_real: + self.context.connectivity.server_address = self._env_vars.cs_server + self.context.connectivity.admin_auth_token = self.api.authentication.xmlrpc_token self.context.resource = mock.MagicMock() self.context.resource.attributes = dict() - self.context.resource.name = self._env_vars.sb_service_alias + self.context.resource.name = self._sb_service_alias self.context.resource.model = 'Generic Terraform Service' - self.set_context_resource_attributes() + if is_api_real: + self.set_context_resource_attributes_from_cs() + self.context.reservation = mock.MagicMock() + self.context.reservation.reservation_id = self._env_vars.cs_res_id + self.context.reservation.domain = self._env_vars.cs_domain + else: + self.context.resource.attributes = ServiceAttributesFactory.create_empty_attributes() + self.context.reservation = mock.MagicMock() - self.context.reservation = mock.MagicMock() - self.context.reservation.reservation_id = self._env_vars.cs_res_id - self.context.reservation.domain = self._env_vars.cs_domain - def set_context_resource_attributes(self): - services = self.real_api.GetReservationDetails(self._env_vars.cs_res_id).ReservationDescription.Services + def set_context_resource_attributes_from_cs(self, the_only_attribute_to_update: str = ""): + services = self.api.GetReservationDetails(self._env_vars.cs_res_id, disableCache=True) \ + .ReservationDescription.Services for service in services: - if service.Alias == self._env_vars.sb_service_alias: + if service.Alias == self._sb_service_alias: for attribute in service.Attributes: - self.context.resource.attributes[attribute.Name] = attribute.Value + if the_only_attribute_to_update and attribute.Name == the_only_attribute_to_update: + self.context.resource.attributes[attribute.Name] = attribute.Value + return + elif not the_only_attribute_to_update: + self.context.resource.attributes[attribute.Name] = attribute.Value + + def create_tf_shell(self): + self._config = TerraformShellConfig(write_sandbox_messages=True, update_live_status=True) + self.tf_shell = TerraformShell(self.context, self._logger, self._config) diff --git a/package/tests/integration_tests/helper_services/service_attributes_factory.py b/package/tests/integration_tests/helper_services/service_attributes_factory.py new file mode 100644 index 00000000..f893e71c --- /dev/null +++ b/package/tests/integration_tests/helper_services/service_attributes_factory.py @@ -0,0 +1,25 @@ +from typing import List + +from cloudshell.api.cloudshell_api import NameValuePair + +from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES + + +class ServiceAttributesFactory: + @staticmethod + def create_empty_attributes() -> List[dict]: + attributes = [ + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.BRANCH}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.CUSTOM_TAGS}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.APPLY_TAGS}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TOKEN}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.CLOUD_PROVIDER}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_OUTPUTS}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_SENSIITVE_OUTPUTS}", Value=""), + NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", Value="") + ] + return attributes diff --git a/package/tests/integration_tests/int_test_downloader.py b/package/tests/integration_tests/int_test_downloader.py index 7fe995fa..60c9e0c3 100644 --- a/package/tests/integration_tests/int_test_downloader.py +++ b/package/tests/integration_tests/int_test_downloader.py @@ -8,7 +8,7 @@ from cloudshell.iac.terraform.models.shell_helper import ShellHelperObject from tests.constants import GITHUB_TF_PUBLIC_HELLO_URL_FILE, GITHUB_TF_PUBLIC_HELLO_URL_FOLDER, TERRAFORM_EXEC_FILE, \ SHELL_NAME, TF_HELLO_FILE -from tests.integration_tests.helper_objects.integration_context import IntegrationData +from tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData from cloudshell.iac.terraform.tagging.tags import TagsManager from cloudshell.iac.terraform.services.svc_attribute_handler import ServiceAttrHandler diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py new file mode 100644 index 00000000..823b0479 --- /dev/null +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -0,0 +1,226 @@ +from unittest.mock import patch, Mock + +# from cloudshell.api.cloudshell_api import NameValuePair +from cloudshell.api.cloudshell_api import NameValuePair +from dotenv import load_dotenv +from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES +from typing import Callable + +from tests.integration_tests.helper_objects.integration_context import IntegrationData + +import os +from unittest import TestCase, mock + +from pathlib import Path + +from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesFactory + + +class TestMockTerraformExecuteDestroy(TestCase): + @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') + def setUp(self, patched_api) -> None: + load_dotenv(Path('int_tests.env')) + + self._prepare_mock_api() + patched_api.return_value.get_api.return_value = self.mock_api + + self._mocked_tf_working_dir = '' + self._prepare_mock_services() + + self.mock_api.GetReservationDetails.return_value.ReservationDescription.Services = [self._service1, + self._service2] + self._prepare_integration_data() + + def _prepare_integration_data(self): + self.integration_data1 = IntegrationData(self._service1.Alias, False, self.mock_api) + self.integration_data2 = IntegrationData(self._service2.Alias, False, self.mock_api) + + def _prepare_mock_services(self): + self._service1 = Mock() + self._service1.Alias = os.environ.get("SB_SERVICE_ALIAS1") + self._service1.Attributes = ServiceAttributesFactory.create_empty_attributes() + self._service2 = Mock() + self._service2.Alias = os.environ.get("SB_SERVICE_ALIAS2") + self._service2.Attributes = ServiceAttributesFactory.create_empty_attributes() + + def _prepare_mock_api(self): + self.mock_api = Mock() + self.mock_api.DecryptPassword = _decrypt_password + self.mock_api.GetResourceDetails.return_value.ResourceFamilyName = 'Cloud Provider' + self.mock_api.GetResourceDetails.return_value.ResourceModelName = 'Microsoft Azure' + self.mock_api.GetResourceDetails.return_value.ResourceAttributes = [ + NameValuePair(Name="Azure Subscription ID", Value=os.environ.get("AZURE_SUBSCRIPTION_ID")), + NameValuePair(Name="Azure Tenant ID", Value=os.environ.get("AZURE_TENANT_ID")), + NameValuePair(Name="Azure Application ID", Value=os.environ.get("AZURE_APPLICATION_ID")), + NameValuePair(Name="Azure Application Key", Value=os.environ.get("AZURE_APPLICATION_KEY_DEC")) + ] + + '''------------------------------ Generic Execute/Destroy functions ---------------------------------''' + + def run_execute(self, pre_exec_function: Callable, integration_data: IntegrationData): + self.pre_exec_prep(pre_exec_function, integration_data) + integration_data.tf_shell.execute_terraform() + + def run_destroy(self, pre_destroy_function: Callable, integration_data: IntegrationData): + self.pre_destroy_prep(pre_destroy_function, integration_data) + integration_data.tf_shell.destroy_terraform() + + def run_execute_and_destroy(self, pre_exec_function: Callable, pre_destroy_function: Callable, + integration_data: IntegrationData): + self.run_execute(pre_exec_function, integration_data) + self.run_destroy(pre_destroy_function, integration_data) + + '''------------------------------ Test Cases ---------------------------------''' + + @patch('cloudshell.iac.terraform.services.tf_proc_exec.TfProcExec.can_destroy_run') + @patch('cloudshell.iac.terraform.terraform_shell.SandboxDataHandler') + @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') + def test_execute_and_destroy_azure_vault(self, patch_api, patched_sbdata_handler, can_destroy_run): + can_destroy_run.return_value = True + patch_api.return_value.get_api.return_value = self.mock_api + mock_sbdata_handler = Mock() + mock_sbdata_handler.get_tf_working_dir = self._get_mocked_tf_working_dir + mock_sbdata_handler.set_tf_working_dir = self._set_mocked_tf_working_dir + patched_sbdata_handler.return_value = mock_sbdata_handler + self.run_execute_and_destroy( + pre_exec_function=self.pre_exec_azure_vault, + pre_destroy_function=self.pre_destroy, + integration_data=self.integration_data1 + ) + + '''------------------------------ Functions : general _pre prep functions ---------------------------------''' + + def pre_exec_prep(self, pre_exec_function: Callable, integration_data: IntegrationData): + pre_exec_function(integration_data) + + def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: IntegrationData): + pre_destroy_function(integration_data) + + def clear_sb_data(self): + self.integration_data1.api.ClearSandboxData(self.integration_data1.context.reservation.reservation_id) + + '''------------------------------ Functions : prep before exec -------------------------------------------''' + + def pre_exec(self, integration_data: IntegrationData): + pass + + def pre_exec_azure_vault(self, integration_data: IntegrationData): + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.Terraform Inputs", + os.environ.get("AZUREAPP_TF_INPUTS"), + integration_data + ) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.Github Terraform Module URL", + os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), + integration_data + ) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TOKEN}", + os.environ.get("GITHUB_TOKEN_DEC"), + integration_data + ) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", + os.environ.get("0.15.1"), + integration_data + ) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.CLOUD_PROVIDER}", + os.environ.get("CLP_RESOURSE"), + integration_data + ) + self._prepare_service1_before_execute(integration_data) + + def _prepare_service1_before_execute(self, integration_data): + service1 = Mock() + service1.Alias = integration_data.context.resource.name + service1.Attributes = integration_data.context.resource.attributes + self.mock_api.GetReservationDetails.return_value.ReservationDescription.Services = [service1] + integration_data.create_tf_shell() + + def pre_exec_azure_mssql(self, integration_data: IntegrationData): + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.Terraform Inputs", + os.environ.get("AZUREMSSQL_TF_INPUTS"), + integration_data + ) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.Github Terraform Module URL", + os.environ.get("GITHUB_TF_PRIVATE_AZUREMSSQL_URL"), + integration_data + ) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.UUID", + os.environ.get(""), + integration_data + ) + self._prepare_service1_before_execute(integration_data) + + def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: IntegrationData): + self.pre_exec_azure_vault(integration_data) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_ACCESS_KEY"), + integration_data + ) + + def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: IntegrationData): + self.pre_exec_azure_vault(integration_data) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_CLOUD_CRED"), + integration_data + ) + + def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: IntegrationData): + self.pre_exec_azure_vault(integration_data) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE"), + integration_data + ) + + def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: IntegrationData): + self.pre_exec_azure_vault(integration_data) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE"), + integration_data + ) + + def pre_exec_azure_vault_without_remote(self, integration_data: IntegrationData): + self.pre_exec_azure_vault(integration_data) + self._set_attribute_on_mock_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get(""), + integration_data + ) + + '''------------------------------ Functions : prep before destroy -----------------------------------------''' + + def pre_destroy(self, integration_data: IntegrationData): + # As UUID has been created and SB data now contains UUID and Status we must update context so destroy can run + for attribute in integration_data.context.resource.attributes: + if attribute.Name == f"{SHELL_NAME}.UUID": + attribute.Value = integration_data.tf_shell._tf_service.attributes[f"{SHELL_NAME}.UUID"] + + '''------------------------------ Helper Functions ---------------------------------------------------------''' + @staticmethod + def _set_attribute_on_mock_service(attr_name: str, attr_value: str, integration_data: IntegrationData): + for attribute in integration_data.context.resource.attributes: + if attribute.Name == attr_name: + attribute.Value = attr_value + return + + def _get_mocked_tf_working_dir(self): + return self._mocked_tf_working_dir + + def _set_mocked_tf_working_dir(self, tf_working_dir: str): + self._mocked_tf_working_dir = tf_working_dir + + +def _decrypt_password(x): + result = mock.MagicMock() + result.Value = x + return result diff --git a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py new file mode 100644 index 00000000..b2800a73 --- /dev/null +++ b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py @@ -0,0 +1,209 @@ +from cloudshell.api.cloudshell_api import AttributeNameValue +from dotenv import load_dotenv +from tests.integration_tests.constants import SHELL_NAME +from typing import Callable + +from tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData + +import os +from unittest import TestCase + + +class TestRealTerraformExecuteDestroy(TestCase): + def setUp(self) -> None: + load_dotenv() + self.integration_data1 = IntegrationData(os.environ.get("SB_SERVICE_ALIAS1"), is_api_real=True) + self.integration_data2 = IntegrationData(os.environ.get("SB_SERVICE_ALIAS2"), is_api_real=True) + + def run_execute_and_destroy( + self, pre_exec_function: Callable, + pre_destroy_function: Callable, + integration_data: IntegrationData + ): + self.clear_sb_data() + self.run_execute(pre_exec_function, integration_data) + self.run_destroy(pre_destroy_function, integration_data) + + def run_execute(self, pre_exec_function: Callable, integration_data: IntegrationData): + self.pre_exec_prep(pre_exec_function, integration_data) + integration_data.tf_shell.execute_terraform() + + def run_destroy(self, pre_destroy_function: Callable, integration_data: IntegrationData): + self.pre_destroy_prep(pre_destroy_function, integration_data) + integration_data.tf_shell.destroy_terraform() + + '''------------------------------ Test Cases ---------------------------------''' + + def test_execute_and_destroy(self): + self.run_execute_and_destroy( + pre_exec_function=self.pre_exec, + pre_destroy_function=self.pre_destroy, + integration_data=self.integration_data1 + ) + + def test_execute_and_destroy_azure_vault(self): + self.run_execute_and_destroy( + pre_exec_function=self.pre_exec_azure_vault, + pre_destroy_function=self.pre_destroy, + integration_data=self.integration_data1 + ) + + def test_execute_dual_mssql(self): + self.clear_sb_data() + try: + self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data1) + except Exception as e: + pass + try: + self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data2) + except Exception as e: + pass + self.run_destroy(pre_destroy_function=self.pre_destroy, integration_data=self.integration_data1) + self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data2) + + def test_execute_and_destroy_azure_vault_with_remote_access_key_based(self): + self.run_execute_and_destroy( + pre_exec_function=self.pre_exec_azure_vault_with_remote_access_key_based, + pre_destroy_function=self.pre_destroy, + integration_data=self.integration_data1 + ) + + def test_execute_and_destroy_azure_vault_with_remote_access_cloud_cred_based(self): + self.run_execute_and_destroy( + pre_exec_function=self.pre_exec_azure_vault_with_remote_cloud_cred_based, + pre_destroy_function=self.pre_destroy, + integration_data=self.integration_data1 + ) + + def test_execute_and_destroy_azure_vault_with_remote_invalid_nonexistent(self): + try: + self.run_execute_and_destroy( + pre_exec_function=self.pre_exec_azure_vault_with_remote_invalid_nonexistent, + pre_destroy_function=self.pre_destroy, + integration_data=self.integration_data1 + ) + except Exception as e: + pass + + def test_execute_and_destroy_azure_vault_with_remote_invalid_wrong(self): + try: + self.run_execute_and_destroy( + pre_exec_function=self.pre_exec_azure_vault_with_remote_invalid_wrong, + pre_destroy_function=self.pre_destroy, + integration_data=self.integration_data1 + ) + except Exception as e: + pass + + def test_execute_and_destroy_azure_vault_without_remote(self): + self.run_execute_and_destroy( + pre_exec_function=self.pre_exec_azure_vault_without_remote, + pre_destroy_function=self.pre_destroy, + integration_data=self.integration_data1 + ) + + '''------------------------------ Functions : general _pre prep functions ---------------------------------''' + + def pre_exec_prep(self, pre_exec_function: Callable, integration_data: IntegrationData): + pre_exec_function(integration_data) + + def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: IntegrationData): + pre_destroy_function(integration_data) + + def clear_sb_data(self): + self.integration_data1.api.ClearSandboxData(self.integration_data1.context.reservation.reservation_id) + + '''------------------------------ Functions : prep before exec -------------------------------------------''' + + def pre_exec(self, integration_data: IntegrationData): + pass + + def pre_exec_azure_vault(self, integration_data: IntegrationData): + self._set_attribute_on_service( + f"{SHELL_NAME}.Terraform Inputs", + os.environ.get("AZUREAPP_TF_INPUTS"), + integration_data + ) + self._set_attribute_on_service( + f"{SHELL_NAME}.Github Terraform Module URL", + os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), + integration_data + ) + self._set_attribute_on_service( + f"{SHELL_NAME}.UUID", + os.environ.get(""), + integration_data + ) + + def pre_exec_azure_mssql(self, integration_data: IntegrationData): + self._set_attribute_on_service( + f"{SHELL_NAME}.Terraform Inputs", + os.environ.get("AZUREMSSQL_TF_INPUTS"), + integration_data + ) + self._set_attribute_on_service( + f"{SHELL_NAME}.Github Terraform Module URL", + os.environ.get("GITHUB_TF_PRIVATE_AZUREMSSQL_URL"), + integration_data + ) + self._set_attribute_on_service( + f"{SHELL_NAME}.UUID", + os.environ.get(""), + integration_data + ) + + def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: IntegrationData): + self.pre_exec_azure_vault(integration_data) + self._set_attribute_on_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_ACCESS_KEY"), + integration_data + ) + + def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: IntegrationData): + self.pre_exec_azure_vault(integration_data) + self._set_attribute_on_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_CLOUD_CRED"), + integration_data + ) + + def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: IntegrationData): + self.pre_exec_azure_vault(integration_data) + self._set_attribute_on_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE"), + integration_data + ) + + def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: IntegrationData): + self.pre_exec_azure_vault(integration_data) + self._set_attribute_on_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE"), + integration_data + ) + + def pre_exec_azure_vault_without_remote(self, integration_data: IntegrationData): + self.pre_exec_azure_vault(integration_data) + self._set_attribute_on_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get(""), + integration_data + ) + + '''------------------------------ Functions : prep before destroy -----------------------------------------''' + + def pre_destroy(self, integration_data: IntegrationData): + # As UUID has been created and SB data now contains UUID and Status we must update context so destroy can run + integration_data.set_context_resource_attributes_from_cs(f"{SHELL_NAME}.UUID") + + '''------------------------------ Helper Functions ---------------------------------------------------------''' + + def _set_attribute_on_service(self, attr_name: str, attr_value: str, integration_data: IntegrationData): + attr_req = [AttributeNameValue(attr_name, attr_value)] + integration_data.api.SetServiceAttributesValues( + integration_data.context.reservation.reservation_id, + integration_data.context.resource.name, + attr_req + ) diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env new file mode 100644 index 00000000..9ad99834 --- /dev/null +++ b/package/tests/integration_tests/int_tests.env @@ -0,0 +1,12 @@ +SB_SERVICE_ALIAS1 = "ooo1" +SB_SERVICE_ALIAS2 = "ooo2" + +AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=alexaz-amd-test,KEYVAULT_RG=alexaz-test-amd,SECRET_NAME=test" +GITHUB_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" +GITHUB_TOKEN_DEC = "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" +CLP_RESOURSE = "real_azure" + +AZURE_SUBSCRIPTION_ID = "ee4ae63d-e845-4ff2-bf15-a508e7db58f8" +AZURE_TENANT_ID = "c55d955e-1ea6-4462-987a-ad33142ddffe" +AZURE_APPLICATION_ID = "9890da5c-d331-4eb4-8bde-52d993a17cbb" +AZURE_APPLICATION_KEY_DEC = "61b6d384-3a6a-42f6-9e49-3f6260990455" diff --git a/package/tests/integration_tests/int_tests.template.env b/package/tests/integration_tests/int_tests.template.env new file mode 100644 index 00000000..8c3b453e --- /dev/null +++ b/package/tests/integration_tests/int_tests.template.env @@ -0,0 +1,13 @@ +SB_SERVICE_ALIAS1 = "" +SB_SERVICE_ALIAS2 = "" + + +AZUREAPP_TF_INPUTS = "" +GITHUB_TF_PRIVATE_AZUREAPP_URL = "" +GITHUB_TOKEN_DEC = "" +CLP_RESOURSE = "" + +AZURE_SUBSCRIPTION_ID = "" +AZURE_TENANT_ID = "" +AZURE_APPLICATION_ID = "" +AZURE_APPLICATION_KEY_DEC = "" diff --git a/shells/backends/azure_tf_backend/tests/integration_tests/int_test_validate.py b/shells/backends/azure_tf_backend/tests/integration_tests/int_test_validate.py index 4eb05e5c..d0a7429d 100644 --- a/shells/backends/azure_tf_backend/tests/integration_tests/int_test_validate.py +++ b/shells/backends/azure_tf_backend/tests/integration_tests/int_test_validate.py @@ -1,6 +1,6 @@ from unittest import TestCase -from tests.integration_tests.helper_objects.integration_context import IntegrationData +from tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData class TestTerraformBackend(TestCase): diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py b/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py index 6da43cbe..73f49b7b 100644 --- a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py +++ b/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py @@ -51,7 +51,6 @@ def _set_context(self, is_api_real: bool): self.context.resource.attributes = ServiceAttributesFactory.create_empty_attributes() self.context.reservation = mock.MagicMock() - def set_context_resource_attributes_from_cs(self, the_only_attribute_to_update: str = ""): services = self.api.GetReservationDetails(self._env_vars.cs_res_id, disableCache=True) \ .ReservationDescription.Services diff --git a/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 83fffc0c..02aa8078 100644 --- a/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -18,7 +18,7 @@ class TestMockTerraformExecuteDestroy(TestCase): @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') def setUp(self, patched_api) -> None: - load_dotenv(Path('.env.int_tests')) + load_dotenv(Path('int_tests.env')) self._prepare_mock_api() patched_api.return_value.get_api.return_value = self.mock_api diff --git a/shells/generic_terraform_service/tests/integration_tests/int_test_real_tf_execute_destroy.py b/shells/generic_terraform_service/tests/integration_tests/int_test_real_tf_execute_destroy.py index 86eab030..b2800a73 100644 --- a/shells/generic_terraform_service/tests/integration_tests/int_test_real_tf_execute_destroy.py +++ b/shells/generic_terraform_service/tests/integration_tests/int_test_real_tf_execute_destroy.py @@ -3,7 +3,7 @@ from tests.integration_tests.constants import SHELL_NAME from typing import Callable -from tests.integration_tests.helper_objects.integration_context import IntegrationData +from tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData import os from unittest import TestCase diff --git a/shells/generic_terraform_service/tests/integration_tests/.env.int_tests b/shells/generic_terraform_service/tests/integration_tests/int_tests.env similarity index 100% rename from shells/generic_terraform_service/tests/integration_tests/.env.int_tests rename to shells/generic_terraform_service/tests/integration_tests/int_tests.env From 4825a39f4e09640e0e2d595e5c33dd460f3ea834 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 24 Sep 2021 16:34:34 -0500 Subject: [PATCH 08/70] lint --- .../integration_tests/helper_objects/integration_context.py | 1 - .../integration_tests/helper_objects/integration_helper.py | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index 6da43cbe..73f49b7b 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -51,7 +51,6 @@ def _set_context(self, is_api_real: bool): self.context.resource.attributes = ServiceAttributesFactory.create_empty_attributes() self.context.reservation = mock.MagicMock() - def set_context_resource_attributes_from_cs(self, the_only_attribute_to_update: str = ""): services = self.api.GetReservationDetails(self._env_vars.cs_res_id, disableCache=True) \ .ReservationDescription.Services diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py b/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py index a7c92d51..0067d99c 100644 --- a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py +++ b/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py @@ -41,7 +41,7 @@ def wipe_uuids(sb: Sandbox): sb = Sandbox() -#print_uuids(sb) -#wipe_uuids(sb) -#data = sb.automation_api.GetSandboxData(sb_id) +# print_uuids(sb) +# wipe_uuids(sb) +# data = sb.automation_api.GetSandboxData(sb_id) print("") From c9f5c13f081f3b6820fd35a6a3ca0be4a125e029 Mon Sep 17 00:00:00 2001 From: qualidan Date: Mon, 27 Sep 2021 13:48:24 -0500 Subject: [PATCH 09/70] fix to .env gitignore and naming for files --- .gitignore | 1 + package/tests/{.env.template.env => .env.template} | 0 .../{int_tests.template.env => int_tests.env.template} | 0 .../azure_tf_backend/tests/{.env.template.env => .env.template} | 0 4 files changed, 1 insertion(+) rename package/tests/{.env.template.env => .env.template} (100%) rename package/tests/integration_tests/{int_tests.template.env => int_tests.env.template} (100%) rename shells/backends/azure_tf_backend/tests/{.env.template.env => .env.template} (100%) diff --git a/.gitignore b/.gitignore index db0d7667..f98d9fc3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ __pycache__/ .idea/ venv/ package/tests/.env +package/tests/integration_tests/.env env/ build/ develop-eggs/ diff --git a/package/tests/.env.template.env b/package/tests/.env.template similarity index 100% rename from package/tests/.env.template.env rename to package/tests/.env.template diff --git a/package/tests/integration_tests/int_tests.template.env b/package/tests/integration_tests/int_tests.env.template similarity index 100% rename from package/tests/integration_tests/int_tests.template.env rename to package/tests/integration_tests/int_tests.env.template diff --git a/shells/backends/azure_tf_backend/tests/.env.template.env b/shells/backends/azure_tf_backend/tests/.env.template similarity index 100% rename from shells/backends/azure_tf_backend/tests/.env.template.env rename to shells/backends/azure_tf_backend/tests/.env.template From e570fe9d15231ec5d932cbef2cdf34a93486b4c4 Mon Sep 17 00:00:00 2001 From: qualidan Date: Mon, 27 Sep 2021 16:55:38 -0500 Subject: [PATCH 10/70] added secrets --- .github/workflows/ci.yml | 8 ++++++++ package/tests/integration_tests/constants.py | 2 ++ .../integration_tests/int_test_mock_tf_execute_destroy.py | 6 +++--- .../tests/integration_tests/constants.py | 3 +++ .../integration_tests/int_test_mock_tf_execute_destroy.py | 6 +++--- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33a3179a..eabfa911 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,14 @@ jobs: run: | python -m unittest - name: Test integration tests for shells/generic_terraform_service with unittest + env: + AZUREAPP_TF_INPUTS: ${{ secrets.AZUREAPP_TF_INPUTS }} + GITHUB_TF_PRIVATE_AZUREAPP_URL: ${{ secrets.GITHUB_TF_PRIVATE_AZUREAPP_URL }} + GITHUB_TOKEN_DEC: ${{ secrets.GITHUB_TOKEN_DEC }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} + AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} run: | pwd cd shells/generic_terraform_service diff --git a/package/tests/integration_tests/constants.py b/package/tests/integration_tests/constants.py index 36e27876..4d0843a2 100644 --- a/package/tests/integration_tests/constants.py +++ b/package/tests/integration_tests/constants.py @@ -1,6 +1,8 @@ SHELL_NAME = "Generic Terraform Service" UUID_ATTRIBUTE = f"{SHELL_NAME}.UUID" +INT_TEST_TF_VER = "0.15.1" +INT_TEST_CLP_RESOURSE = "real_azure" class ATTRIBUTE_NAMES: TF_OUTPUTS = "Terraform Outputs" diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 823b0479..3f92e65e 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -3,7 +3,7 @@ # from cloudshell.api.cloudshell_api import NameValuePair from cloudshell.api.cloudshell_api import NameValuePair from dotenv import load_dotenv -from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES +from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE from typing import Callable from tests.integration_tests.helper_objects.integration_context import IntegrationData @@ -122,12 +122,12 @@ def pre_exec_azure_vault(self, integration_data: IntegrationData): ) self._set_attribute_on_mock_service( f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", - os.environ.get("0.15.1"), + INT_TEST_TF_VER, integration_data ) self._set_attribute_on_mock_service( f"{SHELL_NAME}.{ATTRIBUTE_NAMES.CLOUD_PROVIDER}", - os.environ.get("CLP_RESOURSE"), + INT_TEST_CLP_RESOURSE, integration_data ) self._prepare_service1_before_execute(integration_data) diff --git a/shells/generic_terraform_service/tests/integration_tests/constants.py b/shells/generic_terraform_service/tests/integration_tests/constants.py index 36e27876..de273f34 100644 --- a/shells/generic_terraform_service/tests/integration_tests/constants.py +++ b/shells/generic_terraform_service/tests/integration_tests/constants.py @@ -1,6 +1,8 @@ SHELL_NAME = "Generic Terraform Service" UUID_ATTRIBUTE = f"{SHELL_NAME}.UUID" +INT_TEST_TF_VER = "0.15.1" +INT_TEST_CLP_RESOURSE = "real_azure" class ATTRIBUTE_NAMES: TF_OUTPUTS = "Terraform Outputs" @@ -17,3 +19,4 @@ class ATTRIBUTE_NAMES: BRANCH = "Branch" CLOUD_PROVIDER = "Cloud Provider" UUID = "UUID" + diff --git a/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 02aa8078..13404a41 100644 --- a/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -2,7 +2,7 @@ from cloudshell.api.cloudshell_api import NameValuePair from dotenv import load_dotenv -from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES +from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE from typing import Callable from tests.integration_tests.helper_objects.integration_context import IntegrationData @@ -121,12 +121,12 @@ def pre_exec_azure_vault(self, integration_data: IntegrationData): ) self._set_attribute_on_mock_service( f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", - os.environ.get("0.15.1"), + INT_TEST_TF_VER, integration_data ) self._set_attribute_on_mock_service( f"{SHELL_NAME}.{ATTRIBUTE_NAMES.CLOUD_PROVIDER}", - os.environ.get("CLP_RESOURSE"), + INT_TEST_CLP_RESOURSE, integration_data ) self._prepare_service1_before_execute(integration_data) From 2d47217447006e403dd5e009806ef1e72be9f4b5 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Sep 2021 13:49:58 -0500 Subject: [PATCH 11/70] Update constants.py --- package/tests/integration_tests/constants.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package/tests/integration_tests/constants.py b/package/tests/integration_tests/constants.py index 4d0843a2..70d5920d 100644 --- a/package/tests/integration_tests/constants.py +++ b/package/tests/integration_tests/constants.py @@ -4,6 +4,7 @@ INT_TEST_TF_VER = "0.15.1" INT_TEST_CLP_RESOURSE = "real_azure" + class ATTRIBUTE_NAMES: TF_OUTPUTS = "Terraform Outputs" TF_SENSIITVE_OUTPUTS = "Terraform Sensitive Outputs" @@ -18,4 +19,4 @@ class ATTRIBUTE_NAMES: GITHUB_URL = "Github Terraform Module URL" BRANCH = "Branch" CLOUD_PROVIDER = "Cloud Provider" - UUID = "UUID" + UUID = "UUID" \ No newline at end of file From 35319c8d4feff1d7400a99135fe746fd67fa524c Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Sep 2021 13:52:13 -0500 Subject: [PATCH 12/70] flake8 --- package/tests/integration_tests/constants.py | 2 +- .../tests/integration_tests/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package/tests/integration_tests/constants.py b/package/tests/integration_tests/constants.py index 70d5920d..88fb902f 100644 --- a/package/tests/integration_tests/constants.py +++ b/package/tests/integration_tests/constants.py @@ -19,4 +19,4 @@ class ATTRIBUTE_NAMES: GITHUB_URL = "Github Terraform Module URL" BRANCH = "Branch" CLOUD_PROVIDER = "Cloud Provider" - UUID = "UUID" \ No newline at end of file + UUID = "UUID" diff --git a/shells/generic_terraform_service/tests/integration_tests/constants.py b/shells/generic_terraform_service/tests/integration_tests/constants.py index de273f34..88fb902f 100644 --- a/shells/generic_terraform_service/tests/integration_tests/constants.py +++ b/shells/generic_terraform_service/tests/integration_tests/constants.py @@ -4,6 +4,7 @@ INT_TEST_TF_VER = "0.15.1" INT_TEST_CLP_RESOURSE = "real_azure" + class ATTRIBUTE_NAMES: TF_OUTPUTS = "Terraform Outputs" TF_SENSIITVE_OUTPUTS = "Terraform Sensitive Outputs" @@ -19,4 +20,3 @@ class ATTRIBUTE_NAMES: BRANCH = "Branch" CLOUD_PROVIDER = "Cloud Provider" UUID = "UUID" - From 6c8e26334ae926cdf67c6f5989874b5a91073547 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Sep 2021 13:57:41 -0500 Subject: [PATCH 13/70] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eabfa911..eeac045c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,6 @@ jobs: AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} run: | pwd - cd shells/generic_terraform_service + cd package/tests/integration_tests pwd python -m unittest discover -p "int_test_mock*.py" From 310bf0c3797576c2fc0776210d88708b1399aaec Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Sep 2021 14:10:52 -0500 Subject: [PATCH 14/70] Create __init__.py --- package/tests/integration_tests/helper_services/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 package/tests/integration_tests/helper_services/__init__.py diff --git a/package/tests/integration_tests/helper_services/__init__.py b/package/tests/integration_tests/helper_services/__init__.py new file mode 100644 index 00000000..e69de29b From 5a809faef8de592182eedafdb09bdb4a3cdb6810 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 11:17:14 -0500 Subject: [PATCH 15/70] GITHUB -> GH --- .github/workflows/ci.yml | 4 ++-- package/tests/constants.py | 4 ++-- package/tests/integration_tests/int_test_downloader.py | 8 ++++---- .../integration_tests/int_test_mock_tf_execute_destroy.py | 6 +++--- .../integration_tests/int_test_real_tf_execute_destroy.py | 4 ++-- package/tests/integration_tests/int_tests.env | 4 ++-- package/tests/integration_tests/int_tests.env.template | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eeac045c..091764ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,8 +50,8 @@ jobs: - name: Test integration tests for shells/generic_terraform_service with unittest env: AZUREAPP_TF_INPUTS: ${{ secrets.AZUREAPP_TF_INPUTS }} - GITHUB_TF_PRIVATE_AZUREAPP_URL: ${{ secrets.GITHUB_TF_PRIVATE_AZUREAPP_URL }} - GITHUB_TOKEN_DEC: ${{ secrets.GITHUB_TOKEN_DEC }} + GH_TF_PRIVATE_AZUREAPP_URL: ${{ secrets.GITHUB_TF_PRIVATE_AZUREAPP_URL }} + GH_TOKEN_DEC: ${{ secrets.GITHUB_TOKEN_DEC }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} diff --git a/package/tests/constants.py b/package/tests/constants.py index 410c0529..44960afa 100644 --- a/package/tests/constants.py +++ b/package/tests/constants.py @@ -1,5 +1,5 @@ -GITHUB_TF_PUBLIC_HELLO_URL_FILE = "https://github.com/QualiSystemsLab/CloudShell-Terraform-Shell/blob/master/package/tests/tests_helper_files/tf_modules/hello/hello.tf" -GITHUB_TF_PUBLIC_HELLO_URL_FOLDER = "https://github.com/QualiSystemsLab/CloudShell-Terraform-Shell/tree/master/package/tests/tests_helper_files/tf_modules/hello" +GH_TF_PUBLIC_HELLO_URL_FILE = "https://github.com/QualiSystemsLab/CloudShell-Terraform-Shell/blob/master/package/tests/tests_helper_files/tf_modules/hello/hello.tf" +GH_TF_PUBLIC_HELLO_URL_FOLDER = "https://github.com/QualiSystemsLab/CloudShell-Terraform-Shell/tree/master/package/tests/tests_helper_files/tf_modules/hello" TF_STATE_FILE_STRING = 'tf_state_file_string' diff --git a/package/tests/integration_tests/int_test_downloader.py b/package/tests/integration_tests/int_test_downloader.py index 60c9e0c3..941fb453 100644 --- a/package/tests/integration_tests/int_test_downloader.py +++ b/package/tests/integration_tests/int_test_downloader.py @@ -6,7 +6,7 @@ from cloudshell.iac.terraform.services.sandbox_messages import SandboxMessagesService from shells.generic_terraform_service.src.data_model import GenericTerraformService from cloudshell.iac.terraform.models.shell_helper import ShellHelperObject -from tests.constants import GITHUB_TF_PUBLIC_HELLO_URL_FILE, GITHUB_TF_PUBLIC_HELLO_URL_FOLDER, TERRAFORM_EXEC_FILE, \ +from tests.constants import GH_TF_PUBLIC_HELLO_URL_FILE, GH_TF_PUBLIC_HELLO_URL_FOLDER, TERRAFORM_EXEC_FILE, \ SHELL_NAME, TF_HELLO_FILE from tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData from cloudshell.iac.terraform.tagging.tags import TagsManager @@ -62,9 +62,9 @@ def _test_download_terraform_module(self, url: str, branch: str): self.assertTrue(os.path.exists(os.path.join(tf_workingdir, TF_HELLO_FILE))) def test_public_and_private_hello_dl(self): - self._test_download_terraform_module(GITHUB_TF_PUBLIC_HELLO_URL_FILE, "") - self._test_download_terraform_module(os.environ.get("GITHUB_TF_PRIVATE_HELLO_URL"), "") - self._test_download_terraform_module(GITHUB_TF_PUBLIC_HELLO_URL_FOLDER, "") + self._test_download_terraform_module(GH_TF_PUBLIC_HELLO_URL_FILE, "") + self._test_download_terraform_module(os.environ.get("GH_TF_PRIVATE_HELLO_URL"), "") + self._test_download_terraform_module(GH_TF_PUBLIC_HELLO_URL_FOLDER, "") def test_download_terraform_executable(self): downloader = Downloader(self._driver_helper) diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 3f92e65e..2cc7dfdd 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -112,12 +112,12 @@ def pre_exec_azure_vault(self, integration_data: IntegrationData): ) self._set_attribute_on_mock_service( f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), + os.environ.get("GH_TF_PRIVATE_AZUREAPP_URL"), integration_data ) self._set_attribute_on_mock_service( f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TOKEN}", - os.environ.get("GITHUB_TOKEN_DEC"), + os.environ.get("GH_TOKEN_DEC"), integration_data ) self._set_attribute_on_mock_service( @@ -147,7 +147,7 @@ def pre_exec_azure_mssql(self, integration_data: IntegrationData): ) self._set_attribute_on_mock_service( f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GITHUB_TF_PRIVATE_AZUREMSSQL_URL"), + os.environ.get("GH_TF_PRIVATE_AZUREMSSQL_URL"), integration_data ) self._set_attribute_on_mock_service( diff --git a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py index b2800a73..b62e01b3 100644 --- a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py @@ -126,7 +126,7 @@ def pre_exec_azure_vault(self, integration_data: IntegrationData): ) self._set_attribute_on_service( f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), + os.environ.get("GH_TF_PRIVATE_AZUREAPP_URL"), integration_data ) self._set_attribute_on_service( @@ -143,7 +143,7 @@ def pre_exec_azure_mssql(self, integration_data: IntegrationData): ) self._set_attribute_on_service( f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GITHUB_TF_PRIVATE_AZUREMSSQL_URL"), + os.environ.get("GH_TF_PRIVATE_AZUREMSSQL_URL"), integration_data ) self._set_attribute_on_service( diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env index 9ad99834..7191429c 100644 --- a/package/tests/integration_tests/int_tests.env +++ b/package/tests/integration_tests/int_tests.env @@ -2,8 +2,8 @@ SB_SERVICE_ALIAS1 = "ooo1" SB_SERVICE_ALIAS2 = "ooo2" AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=alexaz-amd-test,KEYVAULT_RG=alexaz-test-amd,SECRET_NAME=test" -GITHUB_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" -GITHUB_TOKEN_DEC = "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" +GH_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" +GH_TOKEN_DEC = "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" CLP_RESOURSE = "real_azure" AZURE_SUBSCRIPTION_ID = "ee4ae63d-e845-4ff2-bf15-a508e7db58f8" diff --git a/package/tests/integration_tests/int_tests.env.template b/package/tests/integration_tests/int_tests.env.template index 8c3b453e..ab6e322a 100644 --- a/package/tests/integration_tests/int_tests.env.template +++ b/package/tests/integration_tests/int_tests.env.template @@ -3,8 +3,8 @@ SB_SERVICE_ALIAS2 = "" AZUREAPP_TF_INPUTS = "" -GITHUB_TF_PRIVATE_AZUREAPP_URL = "" -GITHUB_TOKEN_DEC = "" +GH_TF_PRIVATE_AZUREAPP_URL = "" +GH_TOKEN_DEC = "" CLP_RESOURSE = "" AZURE_SUBSCRIPTION_ID = "" From 896e3129cad7649eb898eb44b977688019ade4f0 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 11:22:39 -0500 Subject: [PATCH 16/70] Update ci.yml --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 091764ee..89c3919a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,7 @@ jobs: AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} run: | + echo ${GH_TF_PRIVATE_AZUREAPP_URL} pwd cd package/tests/integration_tests pwd From e5bb2b0f1458b01b5a287e0df6710645f380fa88 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 11:32:12 -0500 Subject: [PATCH 17/70] Update ci.yml --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89c3919a..9ff369cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,11 @@ jobs: AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} run: | - echo ${GH_TF_PRIVATE_AZUREAPP_URL} + echo "##############################" + echo "GH_TF_PRIVATE_AZUREAPP_URL = $GH_TF_PRIVATE_AZUREAPP_URL" + echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" + echo "$GITHUB_CONTEXT" + echo "##############################" pwd cd package/tests/integration_tests pwd From 0c8a198674a6e121ddac23e1181ceb8c01e2bd1b Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 11:45:11 -0500 Subject: [PATCH 18/70] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ff369cd..9ab922f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - name: Test unit tests with unittest run: | python -m unittest - - name: Test integration tests for shells/generic_terraform_service with unittest + - name: Test integration tests for package/tests/integration_tests env: AZUREAPP_TF_INPUTS: ${{ secrets.AZUREAPP_TF_INPUTS }} GH_TF_PRIVATE_AZUREAPP_URL: ${{ secrets.GITHUB_TF_PRIVATE_AZUREAPP_URL }} From 2cd8c060b3165cddf6a3be2c4e2345af97a82a86 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 11:47:08 -0500 Subject: [PATCH 19/70] Update ci.yml --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ab922f8..eea7eefa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,8 @@ jobs: AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} run: | echo "##############################" - echo "GH_TF_PRIVATE_AZUREAPP_URL = $GH_TF_PRIVATE_AZUREAPP_URL" + echo "GH_TF_PRIVATE_AZUREAPP_URL = $$GH_TF_PRIVATE_AZUREAPP_URL" + echo ${#$GH_TF_PRIVATE_AZUREAPP_URL} echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" echo "$GITHUB_CONTEXT" echo "##############################" From 4857224c15c69f753e28234173e3f796de4c41f7 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 11:52:30 -0500 Subject: [PATCH 20/70] Update ci.yml --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eea7eefa..bfc35bb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,16 +50,16 @@ jobs: - name: Test integration tests for package/tests/integration_tests env: AZUREAPP_TF_INPUTS: ${{ secrets.AZUREAPP_TF_INPUTS }} - GH_TF_PRIVATE_AZUREAPP_URL: ${{ secrets.GITHUB_TF_PRIVATE_AZUREAPP_URL }} - GH_TOKEN_DEC: ${{ secrets.GITHUB_TOKEN_DEC }} + GH_TF_PRIVATE_AZUREAPP_URL: ${{ secrets.GH_TF_PRIVATE_AZUREAPP_URL }} + GH_TOKEN_DEC: ${{ secrets.GH_TOKEN_DEC }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} run: | echo "##############################" - echo "GH_TF_PRIVATE_AZUREAPP_URL = $$GH_TF_PRIVATE_AZUREAPP_URL" - echo ${#$GH_TF_PRIVATE_AZUREAPP_URL} + echo "GH_TF_PRIVATE_AZUREAPP_URL = $GH_TF_PRIVATE_AZUREAPP_URL" + echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" echo "$GITHUB_CONTEXT" echo "##############################" From c1533b067bf4d97c596fdf2710eabd9ce3f8aa79 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 11:57:39 -0500 Subject: [PATCH 21/70] Update int_test_real_tf_execute_destroy.py --- .../integration_tests/int_test_real_tf_execute_destroy.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py index b62e01b3..f8dfddad 100644 --- a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py @@ -129,6 +129,9 @@ def pre_exec_azure_vault(self, integration_data: IntegrationData): os.environ.get("GH_TF_PRIVATE_AZUREAPP_URL"), integration_data ) + print("------------------------------------------") + print(os.environ.get("GH_TF_PRIVATE_AZUREAPP_URL")) + print("------------------------------------------") self._set_attribute_on_service( f"{SHELL_NAME}.UUID", os.environ.get(""), From 21321b01955ad6e9f508fc037361ed8670077572 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 12:07:21 -0500 Subject: [PATCH 22/70] Update int_test_real_tf_execute_destroy.py --- .../integration_tests/int_test_real_tf_execute_destroy.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py index f8dfddad..dd535324 100644 --- a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py @@ -11,7 +11,7 @@ class TestRealTerraformExecuteDestroy(TestCase): def setUp(self) -> None: - load_dotenv() + # load_dotenv() self.integration_data1 = IntegrationData(os.environ.get("SB_SERVICE_ALIAS1"), is_api_real=True) self.integration_data2 = IntegrationData(os.environ.get("SB_SERVICE_ALIAS2"), is_api_real=True) @@ -129,9 +129,6 @@ def pre_exec_azure_vault(self, integration_data: IntegrationData): os.environ.get("GH_TF_PRIVATE_AZUREAPP_URL"), integration_data ) - print("------------------------------------------") - print(os.environ.get("GH_TF_PRIVATE_AZUREAPP_URL")) - print("------------------------------------------") self._set_attribute_on_service( f"{SHELL_NAME}.UUID", os.environ.get(""), From 20610db684a270f95dbf384b5782580460b5cc0d Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 12:08:41 -0500 Subject: [PATCH 23/70] Update int_test_real_tf_execute_destroy.py --- .../tests/integration_tests/int_test_real_tf_execute_destroy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py index dd535324..0da923ab 100644 --- a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py @@ -1,5 +1,5 @@ from cloudshell.api.cloudshell_api import AttributeNameValue -from dotenv import load_dotenv +# from dotenv import load_dotenv from tests.integration_tests.constants import SHELL_NAME from typing import Callable From 1c2429debd7db30b37aa3f4e6213f06f9904c8c3 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 12:18:31 -0500 Subject: [PATCH 24/70] test without load_dotenv --- .../integration_tests/int_test_mock_tf_execute_destroy.py | 4 ++-- .../integration_tests/int_test_real_tf_execute_destroy.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 2cc7dfdd..71c7b8b4 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -2,7 +2,7 @@ # from cloudshell.api.cloudshell_api import NameValuePair from cloudshell.api.cloudshell_api import NameValuePair -from dotenv import load_dotenv +# from dotenv import load_dotenv from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE from typing import Callable @@ -19,7 +19,7 @@ class TestMockTerraformExecuteDestroy(TestCase): @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') def setUp(self, patched_api) -> None: - load_dotenv(Path('int_tests.env')) + # load_dotenv(Path('int_tests.env')) self._prepare_mock_api() patched_api.return_value.get_api.return_value = self.mock_api diff --git a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py index 0da923ab..b62e01b3 100644 --- a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_real_tf_execute_destroy.py @@ -1,5 +1,5 @@ from cloudshell.api.cloudshell_api import AttributeNameValue -# from dotenv import load_dotenv +from dotenv import load_dotenv from tests.integration_tests.constants import SHELL_NAME from typing import Callable @@ -11,7 +11,7 @@ class TestRealTerraformExecuteDestroy(TestCase): def setUp(self) -> None: - # load_dotenv() + load_dotenv() self.integration_data1 = IntegrationData(os.environ.get("SB_SERVICE_ALIAS1"), is_api_real=True) self.integration_data2 = IntegrationData(os.environ.get("SB_SERVICE_ALIAS2"), is_api_real=True) From 35425ceae752e35fbe303a44ad86bf8e313258bd Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 12:20:59 -0500 Subject: [PATCH 25/70] Update int_test_mock_tf_execute_destroy.py --- .../tests/integration_tests/int_test_mock_tf_execute_destroy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 71c7b8b4..adaf9fdf 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -11,7 +11,7 @@ import os from unittest import TestCase, mock -from pathlib import Path +# from pathlib import Path from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesFactory From 5ac02261e7509c7b6dc38b36bc8551744abf18d9 Mon Sep 17 00:00:00 2001 From: Dan Amzulescu <35110114+qualidan@users.noreply.github.com> Date: Thu, 30 Sep 2021 12:27:21 -0500 Subject: [PATCH 26/70] Delete int_tests.env --- package/tests/integration_tests/int_tests.env | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 package/tests/integration_tests/int_tests.env diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env deleted file mode 100644 index 7191429c..00000000 --- a/package/tests/integration_tests/int_tests.env +++ /dev/null @@ -1,12 +0,0 @@ -SB_SERVICE_ALIAS1 = "ooo1" -SB_SERVICE_ALIAS2 = "ooo2" - -AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=alexaz-amd-test,KEYVAULT_RG=alexaz-test-amd,SECRET_NAME=test" -GH_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" -GH_TOKEN_DEC = "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" -CLP_RESOURSE = "real_azure" - -AZURE_SUBSCRIPTION_ID = "ee4ae63d-e845-4ff2-bf15-a508e7db58f8" -AZURE_TENANT_ID = "c55d955e-1ea6-4462-987a-ad33142ddffe" -AZURE_APPLICATION_ID = "9890da5c-d331-4eb4-8bde-52d993a17cbb" -AZURE_APPLICATION_KEY_DEC = "61b6d384-3a6a-42f6-9e49-3f6260990455" From c22c5dd25b99edeff35360e7f370bf41a032c00d Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 12:27:52 -0500 Subject: [PATCH 27/70] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f98d9fc3..6549c974 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,4 @@ cloudshell_config.yml shells/generic_terraform_service/tests/.env tests/.env shells/backends/azure_tf_backend/tests/.env +shells/generic_terraform_service/tests/integration_tests/int_tests.env From 6f9470b294f8f8abe33d43dd676683f6278cfcfb Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 12:34:53 -0500 Subject: [PATCH 28/70] fixes --- .gitignore | 1 + .../integration_tests/int_test_mock_tf_execute_destroy.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 6549c974..f9a8a29c 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,4 @@ shells/generic_terraform_service/tests/.env tests/.env shells/backends/azure_tf_backend/tests/.env shells/generic_terraform_service/tests/integration_tests/int_tests.env +package/tests/integration_tests/int_tests.env diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index adaf9fdf..2cc7dfdd 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -2,7 +2,7 @@ # from cloudshell.api.cloudshell_api import NameValuePair from cloudshell.api.cloudshell_api import NameValuePair -# from dotenv import load_dotenv +from dotenv import load_dotenv from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE from typing import Callable @@ -11,7 +11,7 @@ import os from unittest import TestCase, mock -# from pathlib import Path +from pathlib import Path from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesFactory @@ -19,7 +19,7 @@ class TestMockTerraformExecuteDestroy(TestCase): @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') def setUp(self, patched_api) -> None: - # load_dotenv(Path('int_tests.env')) + load_dotenv(Path('int_tests.env')) self._prepare_mock_api() patched_api.return_value.get_api.return_value = self.mock_api From 937e568611a23a0124f0476f34b05a29c510e392 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 12:50:29 -0500 Subject: [PATCH 29/70] env secrets --- .gitignore | 2 +- .../int_test_mock_tf_execute_destroy.py | 2 ++ package/tests/integration_tests/int_tests.env | 9 +++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 package/tests/integration_tests/int_tests.env diff --git a/.gitignore b/.gitignore index f9a8a29c..93215348 100644 --- a/.gitignore +++ b/.gitignore @@ -66,4 +66,4 @@ shells/generic_terraform_service/tests/.env tests/.env shells/backends/azure_tf_backend/tests/.env shells/generic_terraform_service/tests/integration_tests/int_tests.env -package/tests/integration_tests/int_tests.env +package/tests/integration_tests/int_tests_secrets.env diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 2cc7dfdd..5e94d26f 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -20,6 +20,8 @@ class TestMockTerraformExecuteDestroy(TestCase): @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') def setUp(self, patched_api) -> None: load_dotenv(Path('int_tests.env')) + if os.path.isfile(Path('int_tests_secrets.env')): + load_dotenv(Path('int_tests_secrets.env')) self._prepare_mock_api() patched_api.return_value.get_api.return_value = self.mock_api diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env new file mode 100644 index 00000000..a2b4d21c --- /dev/null +++ b/package/tests/integration_tests/int_tests.env @@ -0,0 +1,9 @@ +SB_SERVICE_ALIAS1 = "ooo1" +SB_SERVICE_ALIAS2 = "ooo2" + +AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=alexaz-amd-test,KEYVAULT_RG=alexaz-test-amd,SECRET_NAME=test" +GH_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" + +AZURE_SUBSCRIPTION_ID = "ee4ae63d-e845-4ff2-bf15-a508e7db58f8" +AZURE_TENANT_ID = "c55d955e-1ea6-4462-987a-ad33142ddffe" +AZURE_APPLICATION_ID = "9890da5c-d331-4eb4-8bde-52d993a17cbb" From 1202d05fc927b4f985ffc3af62aa692ce1aa6d7f Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 12:59:18 -0500 Subject: [PATCH 30/70] Update ci.yml --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfc35bb7..75830ada 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,12 +49,7 @@ jobs: python -m unittest - name: Test integration tests for package/tests/integration_tests env: - AZUREAPP_TF_INPUTS: ${{ secrets.AZUREAPP_TF_INPUTS }} - GH_TF_PRIVATE_AZUREAPP_URL: ${{ secrets.GH_TF_PRIVATE_AZUREAPP_URL }} GH_TOKEN_DEC: ${{ secrets.GH_TOKEN_DEC }} - AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} run: | echo "##############################" From 7f52d64e5e3f5a3c4d8cbce10321facc66b66e2a Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 14:06:05 -0500 Subject: [PATCH 31/70] fix for MagicMock --- .github/workflows/ci.yml | 6 ------ .../integration_tests/int_test_mock_tf_execute_destroy.py | 3 ++- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75830ada..7f66880e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,12 +52,6 @@ jobs: GH_TOKEN_DEC: ${{ secrets.GH_TOKEN_DEC }} AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} run: | - echo "##############################" - echo "GH_TF_PRIVATE_AZUREAPP_URL = $GH_TF_PRIVATE_AZUREAPP_URL" - - echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" - echo "$GITHUB_CONTEXT" - echo "##############################" pwd cd package/tests/integration_tests pwd diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 5e94d26f..72311a58 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -223,6 +223,7 @@ def _set_mocked_tf_working_dir(self, tf_working_dir: str): def _decrypt_password(x): - result = mock.MagicMock() + # result = mock.MagicMock() + result = Mock() result.Value = x return result From 3700e428276864addbeaf4a5caf196e49af9046e Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 14:19:34 -0500 Subject: [PATCH 32/70] Update int_test_mock_tf_execute_destroy.py --- .../tests/integration_tests/int_test_mock_tf_execute_destroy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 72311a58..7aef3d63 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -9,7 +9,7 @@ from tests.integration_tests.helper_objects.integration_context import IntegrationData import os -from unittest import TestCase, mock +from unittest import TestCase from pathlib import Path From 97b81a991954694af8f4e9a3c4eb7845a4e7fba3 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 14:32:50 -0500 Subject: [PATCH 33/70] reservation_id mock in context --- .../integration_tests/helper_objects/integration_context.py | 1 + 1 file changed, 1 insertion(+) diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index 73f49b7b..fde7ce51 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -48,6 +48,7 @@ def _set_context(self, is_api_real: bool): self.context.reservation.reservation_id = self._env_vars.cs_res_id self.context.reservation.domain = self._env_vars.cs_domain else: + self.context.reservation.reservation_id = "mock_reservation_id" self.context.resource.attributes = ServiceAttributesFactory.create_empty_attributes() self.context.reservation = mock.MagicMock() From 653714cb1e6fd84a1e66ad703b16a47c5d9f4be2 Mon Sep 17 00:00:00 2001 From: qualidan Date: Thu, 30 Sep 2021 14:35:34 -0500 Subject: [PATCH 34/70] Update integration_context.py --- .../integration_tests/helper_objects/integration_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index fde7ce51..39a7b29f 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -48,9 +48,9 @@ def _set_context(self, is_api_real: bool): self.context.reservation.reservation_id = self._env_vars.cs_res_id self.context.reservation.domain = self._env_vars.cs_domain else: - self.context.reservation.reservation_id = "mock_reservation_id" self.context.resource.attributes = ServiceAttributesFactory.create_empty_attributes() self.context.reservation = mock.MagicMock() + self.context.reservation.reservation_id = "mock_reservation_id" def set_context_resource_attributes_from_cs(self, the_only_attribute_to_update: str = ""): services = self.api.GetReservationDetails(self._env_vars.cs_res_id, disableCache=True) \ From fad8cf4d60700fb251c0da15e111a2b602e6edf3 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 6 Oct 2021 15:52:58 -0500 Subject: [PATCH 35/70] fix .env in .gitignore --- .gitignore | 10 +++++----- .../{int_tests.env.template => int_tests.template.env} | 0 2 files changed, 5 insertions(+), 5 deletions(-) rename package/tests/integration_tests/{int_tests.env.template => int_tests.template.env} (100%) diff --git a/.gitignore b/.gitignore index 93215348..d1b49dbb 100644 --- a/.gitignore +++ b/.gitignore @@ -62,8 +62,8 @@ docs/_build/ # PyBuilder target/ cloudshell_config.yml -shells/generic_terraform_service/tests/.env -tests/.env -shells/backends/azure_tf_backend/tests/.env -shells/generic_terraform_service/tests/integration_tests/int_tests.env -package/tests/integration_tests/int_tests_secrets.env + +# env files +*.env +!*.template.env +!package/tests/integration_tests/int_tests.env \ No newline at end of file diff --git a/package/tests/integration_tests/int_tests.env.template b/package/tests/integration_tests/int_tests.template.env similarity index 100% rename from package/tests/integration_tests/int_tests.env.template rename to package/tests/integration_tests/int_tests.template.env From 7ccc1dc681c292f99382c6f7c7d7a6e4c1d6e678 Mon Sep 17 00:00:00 2001 From: qualidan Date: Mon, 18 Oct 2021 11:25:22 -0500 Subject: [PATCH 36/70] Update clp_envvar_handler.py --- .../iac/terraform/services/clp_envvar_handler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package/cloudshell/iac/terraform/services/clp_envvar_handler.py b/package/cloudshell/iac/terraform/services/clp_envvar_handler.py index 23859355..95879903 100644 --- a/package/cloudshell/iac/terraform/services/clp_envvar_handler.py +++ b/package/cloudshell/iac/terraform/services/clp_envvar_handler.py @@ -57,11 +57,11 @@ def __init__(self, clp_res_model, clp_resource_attributes, shell_helper): def set_env_vars_based_on_clp(self): for attr in self._clp_resource_attributes: - if self.does_attribute_match(self._clp_res_model, attr, self._shell_helper, "Azure Subscription ID"): + if self.does_attribute_match(self._clp_res_model, attr, "Azure Subscription ID"): os.environ["ARM_SUBSCRIPTION_ID"] = attr.Value - if self.does_attribute_match(self._clp_res_model, attr, self._shell_helper, "Azure Tenant ID"): + if self.does_attribute_match(self._clp_res_model, attr, "Azure Tenant ID"): os.environ["Azure Tenant ID"] = attr.Value - if self.does_attribute_match(self._clp_res_model, attr, self._shell_helper, "Azure Application ID"): + if self.does_attribute_match(self._clp_res_model, attr, "Azure Application ID"): os.environ["ARM_CLIENT_ID"] = attr.Value - if self.does_attribute_match(self._clp_res_model, attr, self._shell_helper, "Azure Application Key", True): + if self.does_attribute_match(self._clp_res_model, attr, "Azure Application Key"): os.environ["ARM_CLIENT_SECRET"] = self._shell_helper.api.DecryptPassword(attr.Value).Value From 1228f5f18292e8f8445b697d79aac1ea2d4c3abf Mon Sep 17 00:00:00 2001 From: qualidan Date: Mon, 18 Oct 2021 11:57:43 -0500 Subject: [PATCH 37/70] Update clp_envvar_handler.py --- package/cloudshell/iac/terraform/services/clp_envvar_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/cloudshell/iac/terraform/services/clp_envvar_handler.py b/package/cloudshell/iac/terraform/services/clp_envvar_handler.py index 95879903..51505dce 100644 --- a/package/cloudshell/iac/terraform/services/clp_envvar_handler.py +++ b/package/cloudshell/iac/terraform/services/clp_envvar_handler.py @@ -60,7 +60,7 @@ def set_env_vars_based_on_clp(self): if self.does_attribute_match(self._clp_res_model, attr, "Azure Subscription ID"): os.environ["ARM_SUBSCRIPTION_ID"] = attr.Value if self.does_attribute_match(self._clp_res_model, attr, "Azure Tenant ID"): - os.environ["Azure Tenant ID"] = attr.Value + os.environ["ARM_TENANT_ID"] = attr.Value if self.does_attribute_match(self._clp_res_model, attr, "Azure Application ID"): os.environ["ARM_CLIENT_ID"] = attr.Value if self.does_attribute_match(self._clp_res_model, attr, "Azure Application Key"): From 95a494b6ef6b287fd306b2ce6cac1b4fa28a56cf Mon Sep 17 00:00:00 2001 From: alexazarh Date: Thu, 21 Oct 2021 16:00:24 -0500 Subject: [PATCH 38/70] cleanups --- .../helper_objects/integration_context.py | 4 +- .../service_attributes_factory.py | 2 +- .../int_test_mock_tf_execute_destroy.py | 7 +- .../azure_tf_backend/tests/.env.template | 23 -- .../tests/integration_tests/__init__.py | 0 .../helper_objects/__init__.py | 0 .../helper_objects/env_vars.py | 14 -- .../helper_objects/integration_context.py | 47 ---- .../integration_tests/int_test_validate.py | 17 -- .../tests/.env.template.env | 32 --- .../tests/integration_tests/__init__.py | 0 .../tests/integration_tests/constants.py | 22 -- .../helper_objects/__init__.py | 0 .../helper_objects/env_vars.py | 13 - .../helper_objects/integration_context.py | 68 ------ .../helper_objects/integration_helper.py | 47 ---- .../service_attributes_factory.py | 25 -- .../int_test_mock_tf_execute_destroy.py | 225 ------------------ .../int_test_real_tf_execute_destroy.py | 209 ---------------- .../tests/integration_tests/int_tests.env | 2 - 20 files changed, 7 insertions(+), 750 deletions(-) delete mode 100644 shells/backends/azure_tf_backend/tests/.env.template delete mode 100644 shells/backends/azure_tf_backend/tests/integration_tests/__init__.py delete mode 100644 shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/__init__.py delete mode 100644 shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/env_vars.py delete mode 100644 shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/integration_context.py delete mode 100644 shells/backends/azure_tf_backend/tests/integration_tests/int_test_validate.py delete mode 100644 shells/generic_terraform_service/tests/.env.template.env delete mode 100644 shells/generic_terraform_service/tests/integration_tests/__init__.py delete mode 100644 shells/generic_terraform_service/tests/integration_tests/constants.py delete mode 100644 shells/generic_terraform_service/tests/integration_tests/helper_objects/__init__.py delete mode 100644 shells/generic_terraform_service/tests/integration_tests/helper_objects/env_vars.py delete mode 100644 shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py delete mode 100644 shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py delete mode 100644 shells/generic_terraform_service/tests/integration_tests/helper_services/service_attributes_factory.py delete mode 100644 shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py delete mode 100644 shells/generic_terraform_service/tests/integration_tests/int_test_real_tf_execute_destroy.py delete mode 100644 shells/generic_terraform_service/tests/integration_tests/int_tests.env diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index 39a7b29f..46909858 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -7,7 +7,7 @@ from cloudshell.shell.core.driver_context import ResourceCommandContext from tests.integration_tests.helper_objects.env_vars import EnvVars -from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesFactory +from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder class IntegrationData(object): @@ -48,7 +48,7 @@ def _set_context(self, is_api_real: bool): self.context.reservation.reservation_id = self._env_vars.cs_res_id self.context.reservation.domain = self._env_vars.cs_domain else: - self.context.resource.attributes = ServiceAttributesFactory.create_empty_attributes() + self.context.resource.attributes = ServiceAttributesMockBuilder.create_empty_attributes() self.context.reservation = mock.MagicMock() self.context.reservation.reservation_id = "mock_reservation_id" diff --git a/package/tests/integration_tests/helper_services/service_attributes_factory.py b/package/tests/integration_tests/helper_services/service_attributes_factory.py index f893e71c..c3f5c5b2 100644 --- a/package/tests/integration_tests/helper_services/service_attributes_factory.py +++ b/package/tests/integration_tests/helper_services/service_attributes_factory.py @@ -5,7 +5,7 @@ from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES -class ServiceAttributesFactory: +class ServiceAttributesMockBuilder: @staticmethod def create_empty_attributes() -> List[dict]: attributes = [ diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 7aef3d63..50b8eeed 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -13,10 +13,11 @@ from pathlib import Path -from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesFactory +from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder class TestMockTerraformExecuteDestroy(TestCase): + @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') def setUp(self, patched_api) -> None: load_dotenv(Path('int_tests.env')) @@ -40,10 +41,10 @@ def _prepare_integration_data(self): def _prepare_mock_services(self): self._service1 = Mock() self._service1.Alias = os.environ.get("SB_SERVICE_ALIAS1") - self._service1.Attributes = ServiceAttributesFactory.create_empty_attributes() + self._service1.Attributes = ServiceAttributesMockBuilder.create_empty_attributes() self._service2 = Mock() self._service2.Alias = os.environ.get("SB_SERVICE_ALIAS2") - self._service2.Attributes = ServiceAttributesFactory.create_empty_attributes() + self._service2.Attributes = ServiceAttributesMockBuilder.create_empty_attributes() def _prepare_mock_api(self): self.mock_api = Mock() diff --git a/shells/backends/azure_tf_backend/tests/.env.template b/shells/backends/azure_tf_backend/tests/.env.template deleted file mode 100644 index e1686808..00000000 --- a/shells/backends/azure_tf_backend/tests/.env.template +++ /dev/null @@ -1,23 +0,0 @@ -SB_SERVICE_ALIAS = "" -RESERVATION_ID = "" - -CS_SERVER = "" -CS_USERNAME = "" -CS_PASSWORD = "" - -CLP_RESOURSE = "" -SB_AZURE_TF_BACKEND = "" - -GITHUB_TOKEN_ENC = "" - -TFEXEC_VERSION = "" -RESERVATION_DOMAIN = "" - -GITHUB_TF_PRIVATE_HELLO_URL = "" -GITHUB_TF_PRIVATE_AZUREAPP_URL = "" -AZUREAPP_TF_INPUTS = "" - -# OPTIONAL - -GITHUB_TF_PRIVATE_VAULT_URL = "" -CS_SERVER2 = "" \ No newline at end of file diff --git a/shells/backends/azure_tf_backend/tests/integration_tests/__init__.py b/shells/backends/azure_tf_backend/tests/integration_tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/__init__.py b/shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/env_vars.py b/shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/env_vars.py deleted file mode 100644 index 66baac53..00000000 --- a/shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/env_vars.py +++ /dev/null @@ -1,14 +0,0 @@ -import os - -from dotenv import load_dotenv - - -class EnvVars(object): - def __init__(self): - load_dotenv() - self.cs_user = os.environ.get("CS_USERNAME") - self.cs_pass = os.environ.get("CS_PASSWORD") - self.cs_server = os.environ.get("CS_SERVER") - self.cs_domain = os.environ.get("RESERVATION_DOMAIN") - self.cs_res_id = os.environ.get("RESERVATION_ID") - self.cs_resource = os.environ.get("SB_AZURE_TF_BACKEND") diff --git a/shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/integration_context.py b/shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/integration_context.py deleted file mode 100644 index 8d0e433f..00000000 --- a/shells/backends/azure_tf_backend/tests/integration_tests/helper_objects/integration_context.py +++ /dev/null @@ -1,47 +0,0 @@ -from unittest import mock - -from cloudshell.api.cloudshell_api import CloudShellAPISession -from cloudshell.logging.qs_logger import get_qs_logger -from cloudshell.shell.core.driver_context import ResourceCommandContext - -from tests.integration_tests.helper_objects.env_vars import EnvVars - -from driver import AzureTfBackendDriver - - -class IntegrationData(object): - def __init__(self): - self._env_vars = EnvVars() - self.real_api = CloudShellAPISession( - self._env_vars.cs_server, - self._env_vars.cs_user, - self._env_vars.cs_pass, - self._env_vars.cs_domain - ) - self._set_context() - self._logger = get_qs_logger(log_group=self.context.resource.name) - - self._create_driver() - - def _set_context(self): - self.context = mock.create_autospec(ResourceCommandContext) - self.context.connectivity = mock.MagicMock() - self.context.connectivity.server_address = self._env_vars.cs_server - self.context.connectivity.admin_auth_token = self.real_api.authentication.xmlrpc_token - - self.context.resource = mock.MagicMock() - self.context.resource.attributes = dict() - self.context.resource.name = self._env_vars.cs_resource - self.set_context_resource_attributes() - - self.context.reservation = mock.MagicMock() - self.context.reservation.reservation_id = self._env_vars.cs_res_id - self.context.reservation.domain = self._env_vars.cs_domain - - def set_context_resource_attributes(self): - for attribute in self.real_api.GetResourceDetails(self.context.resource.name).ResourceAttributes: - self.context.resource.attributes[attribute.Name] = attribute.Value - - def _create_driver(self): - self.driver = AzureTfBackendDriver() - self.driver.initialize(self.context) diff --git a/shells/backends/azure_tf_backend/tests/integration_tests/int_test_validate.py b/shells/backends/azure_tf_backend/tests/integration_tests/int_test_validate.py deleted file mode 100644 index d0a7429d..00000000 --- a/shells/backends/azure_tf_backend/tests/integration_tests/int_test_validate.py +++ /dev/null @@ -1,17 +0,0 @@ -from unittest import TestCase - -from tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData - - -class TestTerraformBackend(TestCase): - def setUp(self) -> None: - self.integration_data = IntegrationData() - - def test_get_inventory(self): - self.integration_data.driver.get_inventory(self.integration_data.context) - - def test_get_cfg_get_backend_data(self): - data = self.integration_data.driver.get_backend_data(self.integration_data.context, "UNIQUE_STRING") - - def test_delete_tfstate_file(self): - self.integration_data.driver.delete_tfstate_file(self.integration_data.context, "d") diff --git a/shells/generic_terraform_service/tests/.env.template.env b/shells/generic_terraform_service/tests/.env.template.env deleted file mode 100644 index 248a0fc3..00000000 --- a/shells/generic_terraform_service/tests/.env.template.env +++ /dev/null @@ -1,32 +0,0 @@ -SB_SERVICE_ALIAS1 = "" -SB_SERVICE_ALIAS2 = "" -RESERVATION_ID = "" - -CS_SERVER = "" -CS_USERNAME = "" -CS_PASSWORD = "" - -CLP_RESOURSE = "" -CLP_ATTRIBUTES = "" - -GITHUB_TOKEN_ENC = "" -GITHUB_TOKEN_DEC = "" - -TFEXEC_VERSION = "" -RESERVATION_DOMAIN = "" - -GITHUB_TF_PRIVATE_HELLO_URL = "" -GITHUB_TF_PRIVATE_AZUREAPP_URL = "" -GITHUB_TF_PRIVATE_AZUREMSSQL_URL = "" - -AZUREAPP_TF_INPUTS = "" -AZUREMSSQL_TF_INPUTS = "" - -REMOTE_STATE_PROVIDER_ACCESS_KEY = "" -REMOTE_STATE_PROVIDER_CLOUD_CRED = "" -REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE = "" -REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE = "" -# OPTIONAL - -GITHUB_TF_PRIVATE_VAULT_URL = "" -CS_SERVER2 = "" \ No newline at end of file diff --git a/shells/generic_terraform_service/tests/integration_tests/__init__.py b/shells/generic_terraform_service/tests/integration_tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/shells/generic_terraform_service/tests/integration_tests/constants.py b/shells/generic_terraform_service/tests/integration_tests/constants.py deleted file mode 100644 index 88fb902f..00000000 --- a/shells/generic_terraform_service/tests/integration_tests/constants.py +++ /dev/null @@ -1,22 +0,0 @@ -SHELL_NAME = "Generic Terraform Service" -UUID_ATTRIBUTE = f"{SHELL_NAME}.UUID" - -INT_TEST_TF_VER = "0.15.1" -INT_TEST_CLP_RESOURSE = "real_azure" - - -class ATTRIBUTE_NAMES: - TF_OUTPUTS = "Terraform Outputs" - TF_SENSIITVE_OUTPUTS = "Terraform Sensitive Outputs" - TF_INPUTS = "Terraform Inputs" - CT_INPUTS = "Custom Tags" - APPLY_TAGS = "Apply Tags" - CUSTOM_TAGS = "Custom Tags" - REMOTE_STATE_PROVIDER = "Remote State Provider" - GITHUB_TERRAFORM_MODULE_URL = "Github Terraform Module URL" - TERRAFORM_VERSION = "Terraform Version" - GITHUB_TOKEN = "Github Token" - GITHUB_URL = "Github Terraform Module URL" - BRANCH = "Branch" - CLOUD_PROVIDER = "Cloud Provider" - UUID = "UUID" diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_objects/__init__.py b/shells/generic_terraform_service/tests/integration_tests/helper_objects/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_objects/env_vars.py b/shells/generic_terraform_service/tests/integration_tests/helper_objects/env_vars.py deleted file mode 100644 index 5f99f198..00000000 --- a/shells/generic_terraform_service/tests/integration_tests/helper_objects/env_vars.py +++ /dev/null @@ -1,13 +0,0 @@ -import os - -from dotenv import load_dotenv - - -class EnvVars(object): - def __init__(self): - load_dotenv() - self.cs_user = os.environ.get("CS_USERNAME") - self.cs_pass = os.environ.get("CS_PASSWORD") - self.cs_server = os.environ.get("CS_SERVER") - self.cs_domain = os.environ.get("RESERVATION_DOMAIN") - self.cs_res_id = os.environ.get("RESERVATION_ID") diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py b/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py deleted file mode 100644 index 73f49b7b..00000000 --- a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_context.py +++ /dev/null @@ -1,68 +0,0 @@ -from unittest import mock -from unittest.mock import Mock - -from cloudshell.api.cloudshell_api import CloudShellAPISession -from cloudshell.iac.terraform import TerraformShell, TerraformShellConfig -from cloudshell.logging.qs_logger import get_qs_logger -from cloudshell.shell.core.driver_context import ResourceCommandContext - -from tests.integration_tests.helper_objects.env_vars import EnvVars -from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesFactory - - -class IntegrationData(object): - def __init__(self, service_name: str, is_api_real: bool = True, mock_api: Mock = None): - # - self._sb_service_alias = service_name - - if is_api_real: - self._env_vars = EnvVars() - self.api = CloudShellAPISession( - self._env_vars.cs_server, - self._env_vars.cs_user, - self._env_vars.cs_pass, - self._env_vars.cs_domain - ) - - else: - self.api = mock_api - self.api.authentication.xmlrpc_token = Mock() - self._set_context(is_api_real) - self._logger = get_qs_logger(log_group=self.context.resource.name) - self.create_tf_shell() - - def _set_context(self, is_api_real: bool): - self.context = mock.create_autospec(ResourceCommandContext) - self.context.connectivity = mock.MagicMock() - if is_api_real: - self.context.connectivity.server_address = self._env_vars.cs_server - self.context.connectivity.admin_auth_token = self.api.authentication.xmlrpc_token - - self.context.resource = mock.MagicMock() - self.context.resource.attributes = dict() - self.context.resource.name = self._sb_service_alias - self.context.resource.model = 'Generic Terraform Service' - if is_api_real: - self.set_context_resource_attributes_from_cs() - self.context.reservation = mock.MagicMock() - self.context.reservation.reservation_id = self._env_vars.cs_res_id - self.context.reservation.domain = self._env_vars.cs_domain - else: - self.context.resource.attributes = ServiceAttributesFactory.create_empty_attributes() - self.context.reservation = mock.MagicMock() - - def set_context_resource_attributes_from_cs(self, the_only_attribute_to_update: str = ""): - services = self.api.GetReservationDetails(self._env_vars.cs_res_id, disableCache=True) \ - .ReservationDescription.Services - for service in services: - if service.Alias == self._sb_service_alias: - for attribute in service.Attributes: - if the_only_attribute_to_update and attribute.Name == the_only_attribute_to_update: - self.context.resource.attributes[attribute.Name] = attribute.Value - return - elif not the_only_attribute_to_update: - self.context.resource.attributes[attribute.Name] = attribute.Value - - def create_tf_shell(self): - self._config = TerraformShellConfig(write_sandbox_messages=True, update_live_status=True) - self.tf_shell = TerraformShell(self.context, self._logger, self._config) diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py b/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py deleted file mode 100644 index 0067d99c..00000000 --- a/shells/generic_terraform_service/tests/integration_tests/helper_objects/integration_helper.py +++ /dev/null @@ -1,47 +0,0 @@ -import os -import cloudshell.helpers.scripts.cloudshell_dev_helpers as dev_helpers -from cloudshell.api.cloudshell_api import AttributeNameValue -from cloudshell.workflow.orchestration.sandbox import Sandbox -from tests.integration_tests.constants import SHELL_NAME, UUID_ATTRIBUTE - -from dotenv import load_dotenv - -load_dotenv() - -server_address = os.environ.get("CS_SERVER") -sb_id = os.environ.get("RESERVATION_ID") -cs_user = os.environ.get("CS_USERNAME") -cs_pass = os.environ.get("CS_PASSWORD") -cs_domain = os.environ.get("RESERVATION_DOMAIN") - - -dev_helpers.attach_to_cloudshell_as(cs_user, cs_pass, cs_domain, sb_id, server_address=server_address, cloudshell_api_port='8029') - - -def print_uuids(sb: Sandbox): - services = sb.automation_api.GetReservationDetails(sb_id, disableCache=True).ReservationDescription.Services - for service in services: - for attribute in service.Attributes: - if attribute.Name == f"{SHELL_NAME}.UUID": - service_data = "" - data = sb.automation_api.GetSandboxData(sb_id) - for sbdkv in data.SandboxDataKeyValues: - if sbdkv.Key == attribute.Value: - service_data = sbdkv.Value - print(f"{service.Alias}.UUID={attribute.Value} \n data={service_data}\n\n") - continue - - -def wipe_uuids(sb: Sandbox): - services = sb.automation_api.GetReservationDetails(sb_id, disableCache=True).ReservationDescription.Services - attr_req = [AttributeNameValue(UUID_ATTRIBUTE, "")] - - for service in services: - sb.automation_api.SetServiceAttributesValues(sb_id, service.Alias, attr_req) - - -sb = Sandbox() -# print_uuids(sb) -# wipe_uuids(sb) -# data = sb.automation_api.GetSandboxData(sb_id) -print("") diff --git a/shells/generic_terraform_service/tests/integration_tests/helper_services/service_attributes_factory.py b/shells/generic_terraform_service/tests/integration_tests/helper_services/service_attributes_factory.py deleted file mode 100644 index f893e71c..00000000 --- a/shells/generic_terraform_service/tests/integration_tests/helper_services/service_attributes_factory.py +++ /dev/null @@ -1,25 +0,0 @@ -from typing import List - -from cloudshell.api.cloudshell_api import NameValuePair - -from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES - - -class ServiceAttributesFactory: - @staticmethod - def create_empty_attributes() -> List[dict]: - attributes = [ - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.BRANCH}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.CUSTOM_TAGS}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.APPLY_TAGS}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TOKEN}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.CLOUD_PROVIDER}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_OUTPUTS}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_SENSIITVE_OUTPUTS}", Value=""), - NameValuePair(Name=f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", Value="") - ] - return attributes diff --git a/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py deleted file mode 100644 index 13404a41..00000000 --- a/shells/generic_terraform_service/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ /dev/null @@ -1,225 +0,0 @@ -from unittest.mock import patch, Mock - -from cloudshell.api.cloudshell_api import NameValuePair -from dotenv import load_dotenv -from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE -from typing import Callable - -from tests.integration_tests.helper_objects.integration_context import IntegrationData - -import os -from unittest import TestCase, mock - -from pathlib import Path - -from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesFactory - - -class TestMockTerraformExecuteDestroy(TestCase): - @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') - def setUp(self, patched_api) -> None: - load_dotenv(Path('int_tests.env')) - - self._prepare_mock_api() - patched_api.return_value.get_api.return_value = self.mock_api - - self._mocked_tf_working_dir = '' - self._prepare_mock_services() - - self.mock_api.GetReservationDetails.return_value.ReservationDescription.Services = [self._service1, - self._service2] - self._prepare_integration_data() - - def _prepare_integration_data(self): - self.integration_data1 = IntegrationData(self._service1.Alias, False, self.mock_api) - self.integration_data2 = IntegrationData(self._service2.Alias, False, self.mock_api) - - def _prepare_mock_services(self): - self._service1 = Mock() - self._service1.Alias = os.environ.get("SB_SERVICE_ALIAS1") - self._service1.Attributes = ServiceAttributesFactory.create_empty_attributes() - self._service2 = Mock() - self._service2.Alias = os.environ.get("SB_SERVICE_ALIAS2") - self._service2.Attributes = ServiceAttributesFactory.create_empty_attributes() - - def _prepare_mock_api(self): - self.mock_api = Mock() - self.mock_api.DecryptPassword = _decrypt_password - self.mock_api.GetResourceDetails.return_value.ResourceFamilyName = 'Cloud Provider' - self.mock_api.GetResourceDetails.return_value.ResourceModelName = 'Microsoft Azure' - self.mock_api.GetResourceDetails.return_value.ResourceAttributes = [ - NameValuePair(Name="Azure Subscription ID", Value=os.environ.get("AZURE_SUBSCRIPTION_ID")), - NameValuePair(Name="Azure Tenant ID", Value=os.environ.get("AZURE_TENANT_ID")), - NameValuePair(Name="Azure Application ID", Value=os.environ.get("AZURE_APPLICATION_ID")), - NameValuePair(Name="Azure Application Key", Value=os.environ.get("AZURE_APPLICATION_KEY_DEC")) - ] - - '''------------------------------ Generic Execute/Destroy functions ---------------------------------''' - - def run_execute(self, pre_exec_function: Callable, integration_data: IntegrationData): - self.pre_exec_prep(pre_exec_function, integration_data) - integration_data.tf_shell.execute_terraform() - - def run_destroy(self, pre_destroy_function: Callable, integration_data: IntegrationData): - self.pre_destroy_prep(pre_destroy_function, integration_data) - integration_data.tf_shell.destroy_terraform() - - def run_execute_and_destroy(self, pre_exec_function: Callable, pre_destroy_function: Callable, - integration_data: IntegrationData): - self.run_execute(pre_exec_function, integration_data) - self.run_destroy(pre_destroy_function, integration_data) - - '''------------------------------ Test Cases ---------------------------------''' - - @patch('cloudshell.iac.terraform.services.tf_proc_exec.TfProcExec.can_destroy_run') - @patch('cloudshell.iac.terraform.terraform_shell.SandboxDataHandler') - @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') - def test_execute_and_destroy_azure_vault(self, patch_api, patched_sbdata_handler, can_destroy_run): - can_destroy_run.return_value = True - patch_api.return_value.get_api.return_value = self.mock_api - mock_sbdata_handler = Mock() - mock_sbdata_handler.get_tf_working_dir = self._get_mocked_tf_working_dir - mock_sbdata_handler.set_tf_working_dir = self._set_mocked_tf_working_dir - patched_sbdata_handler.return_value = mock_sbdata_handler - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - - '''------------------------------ Functions : general _pre prep functions ---------------------------------''' - - def pre_exec_prep(self, pre_exec_function: Callable, integration_data: IntegrationData): - pre_exec_function(integration_data) - - def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: IntegrationData): - pre_destroy_function(integration_data) - - def clear_sb_data(self): - self.integration_data1.api.ClearSandboxData(self.integration_data1.context.reservation.reservation_id) - - '''------------------------------ Functions : prep before exec -------------------------------------------''' - - def pre_exec(self, integration_data: IntegrationData): - pass - - def pre_exec_azure_vault(self, integration_data: IntegrationData): - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Terraform Inputs", - os.environ.get("AZUREAPP_TF_INPUTS"), - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TOKEN}", - os.environ.get("GITHUB_TOKEN_DEC"), - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", - INT_TEST_TF_VER, - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.CLOUD_PROVIDER}", - INT_TEST_CLP_RESOURSE, - integration_data - ) - self._prepare_service1_before_execute(integration_data) - - def _prepare_service1_before_execute(self, integration_data): - service1 = Mock() - service1.Alias = integration_data.context.resource.name - service1.Attributes = integration_data.context.resource.attributes - self.mock_api.GetReservationDetails.return_value.ReservationDescription.Services = [service1] - integration_data.create_tf_shell() - - def pre_exec_azure_mssql(self, integration_data: IntegrationData): - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Terraform Inputs", - os.environ.get("AZUREMSSQL_TF_INPUTS"), - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GITHUB_TF_PRIVATE_AZUREMSSQL_URL"), - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.UUID", - os.environ.get(""), - integration_data - ) - self._prepare_service1_before_execute(integration_data) - - def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: IntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_ACCESS_KEY"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: IntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_CLOUD_CRED"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: IntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: IntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE"), - integration_data - ) - - def pre_exec_azure_vault_without_remote(self, integration_data: IntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get(""), - integration_data - ) - - '''------------------------------ Functions : prep before destroy -----------------------------------------''' - - def pre_destroy(self, integration_data: IntegrationData): - # As UUID has been created and SB data now contains UUID and Status we must update context so destroy can run - for attribute in integration_data.context.resource.attributes: - if attribute.Name == f"{SHELL_NAME}.UUID": - attribute.Value = integration_data.tf_shell._tf_service.attributes[f"{SHELL_NAME}.UUID"] - - '''------------------------------ Helper Functions ---------------------------------------------------------''' - @staticmethod - def _set_attribute_on_mock_service(attr_name: str, attr_value: str, integration_data: IntegrationData): - for attribute in integration_data.context.resource.attributes: - if attribute.Name == attr_name: - attribute.Value = attr_value - return - - def _get_mocked_tf_working_dir(self): - return self._mocked_tf_working_dir - - def _set_mocked_tf_working_dir(self, tf_working_dir: str): - self._mocked_tf_working_dir = tf_working_dir - - -def _decrypt_password(x): - result = mock.MagicMock() - result.Value = x - return result diff --git a/shells/generic_terraform_service/tests/integration_tests/int_test_real_tf_execute_destroy.py b/shells/generic_terraform_service/tests/integration_tests/int_test_real_tf_execute_destroy.py deleted file mode 100644 index b2800a73..00000000 --- a/shells/generic_terraform_service/tests/integration_tests/int_test_real_tf_execute_destroy.py +++ /dev/null @@ -1,209 +0,0 @@ -from cloudshell.api.cloudshell_api import AttributeNameValue -from dotenv import load_dotenv -from tests.integration_tests.constants import SHELL_NAME -from typing import Callable - -from tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData - -import os -from unittest import TestCase - - -class TestRealTerraformExecuteDestroy(TestCase): - def setUp(self) -> None: - load_dotenv() - self.integration_data1 = IntegrationData(os.environ.get("SB_SERVICE_ALIAS1"), is_api_real=True) - self.integration_data2 = IntegrationData(os.environ.get("SB_SERVICE_ALIAS2"), is_api_real=True) - - def run_execute_and_destroy( - self, pre_exec_function: Callable, - pre_destroy_function: Callable, - integration_data: IntegrationData - ): - self.clear_sb_data() - self.run_execute(pre_exec_function, integration_data) - self.run_destroy(pre_destroy_function, integration_data) - - def run_execute(self, pre_exec_function: Callable, integration_data: IntegrationData): - self.pre_exec_prep(pre_exec_function, integration_data) - integration_data.tf_shell.execute_terraform() - - def run_destroy(self, pre_destroy_function: Callable, integration_data: IntegrationData): - self.pre_destroy_prep(pre_destroy_function, integration_data) - integration_data.tf_shell.destroy_terraform() - - '''------------------------------ Test Cases ---------------------------------''' - - def test_execute_and_destroy(self): - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - - def test_execute_and_destroy_azure_vault(self): - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - - def test_execute_dual_mssql(self): - self.clear_sb_data() - try: - self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data1) - except Exception as e: - pass - try: - self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data2) - except Exception as e: - pass - self.run_destroy(pre_destroy_function=self.pre_destroy, integration_data=self.integration_data1) - self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data2) - - def test_execute_and_destroy_azure_vault_with_remote_access_key_based(self): - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault_with_remote_access_key_based, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - - def test_execute_and_destroy_azure_vault_with_remote_access_cloud_cred_based(self): - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault_with_remote_cloud_cred_based, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - - def test_execute_and_destroy_azure_vault_with_remote_invalid_nonexistent(self): - try: - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault_with_remote_invalid_nonexistent, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - except Exception as e: - pass - - def test_execute_and_destroy_azure_vault_with_remote_invalid_wrong(self): - try: - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault_with_remote_invalid_wrong, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - except Exception as e: - pass - - def test_execute_and_destroy_azure_vault_without_remote(self): - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault_without_remote, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - - '''------------------------------ Functions : general _pre prep functions ---------------------------------''' - - def pre_exec_prep(self, pre_exec_function: Callable, integration_data: IntegrationData): - pre_exec_function(integration_data) - - def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: IntegrationData): - pre_destroy_function(integration_data) - - def clear_sb_data(self): - self.integration_data1.api.ClearSandboxData(self.integration_data1.context.reservation.reservation_id) - - '''------------------------------ Functions : prep before exec -------------------------------------------''' - - def pre_exec(self, integration_data: IntegrationData): - pass - - def pre_exec_azure_vault(self, integration_data: IntegrationData): - self._set_attribute_on_service( - f"{SHELL_NAME}.Terraform Inputs", - os.environ.get("AZUREAPP_TF_INPUTS"), - integration_data - ) - self._set_attribute_on_service( - f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), - integration_data - ) - self._set_attribute_on_service( - f"{SHELL_NAME}.UUID", - os.environ.get(""), - integration_data - ) - - def pre_exec_azure_mssql(self, integration_data: IntegrationData): - self._set_attribute_on_service( - f"{SHELL_NAME}.Terraform Inputs", - os.environ.get("AZUREMSSQL_TF_INPUTS"), - integration_data - ) - self._set_attribute_on_service( - f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GITHUB_TF_PRIVATE_AZUREMSSQL_URL"), - integration_data - ) - self._set_attribute_on_service( - f"{SHELL_NAME}.UUID", - os.environ.get(""), - integration_data - ) - - def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: IntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_ACCESS_KEY"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: IntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_CLOUD_CRED"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: IntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: IntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE"), - integration_data - ) - - def pre_exec_azure_vault_without_remote(self, integration_data: IntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get(""), - integration_data - ) - - '''------------------------------ Functions : prep before destroy -----------------------------------------''' - - def pre_destroy(self, integration_data: IntegrationData): - # As UUID has been created and SB data now contains UUID and Status we must update context so destroy can run - integration_data.set_context_resource_attributes_from_cs(f"{SHELL_NAME}.UUID") - - '''------------------------------ Helper Functions ---------------------------------------------------------''' - - def _set_attribute_on_service(self, attr_name: str, attr_value: str, integration_data: IntegrationData): - attr_req = [AttributeNameValue(attr_name, attr_value)] - integration_data.api.SetServiceAttributesValues( - integration_data.context.reservation.reservation_id, - integration_data.context.resource.name, - attr_req - ) diff --git a/shells/generic_terraform_service/tests/integration_tests/int_tests.env b/shells/generic_terraform_service/tests/integration_tests/int_tests.env deleted file mode 100644 index 570dec65..00000000 --- a/shells/generic_terraform_service/tests/integration_tests/int_tests.env +++ /dev/null @@ -1,2 +0,0 @@ -SB_SERVICE_ALIAS1 = "ooo1" -SB_SERVICE_ALIAS2 = "ooo2" From 28b5e30f8d619a54796196652771b1afc8e7e6ba Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 22 Oct 2021 13:06:21 -0500 Subject: [PATCH 39/70] moved azure ID`s to secrets --- .github/workflows/ci.yml | 4 ++++ package/tests/integration_tests/int_tests.env | 6 +----- .../tests/integration_tests/int_tests.template.env | 13 ------------- .../int_tests_secrets.template.env | 6 ++++++ 4 files changed, 11 insertions(+), 18 deletions(-) delete mode 100644 package/tests/integration_tests/int_tests.template.env create mode 100644 package/tests/integration_tests/int_tests_secrets.template.env diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f66880e..9e64ef9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,10 @@ jobs: env: GH_TOKEN_DEC: ${{ secrets.GH_TOKEN_DEC }} AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} + run: | pwd cd package/tests/integration_tests diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env index a2b4d21c..76a10261 100644 --- a/package/tests/integration_tests/int_tests.env +++ b/package/tests/integration_tests/int_tests.env @@ -2,8 +2,4 @@ SB_SERVICE_ALIAS1 = "ooo1" SB_SERVICE_ALIAS2 = "ooo2" AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=alexaz-amd-test,KEYVAULT_RG=alexaz-test-amd,SECRET_NAME=test" -GH_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" - -AZURE_SUBSCRIPTION_ID = "ee4ae63d-e845-4ff2-bf15-a508e7db58f8" -AZURE_TENANT_ID = "c55d955e-1ea6-4462-987a-ad33142ddffe" -AZURE_APPLICATION_ID = "9890da5c-d331-4eb4-8bde-52d993a17cbb" +GH_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" \ No newline at end of file diff --git a/package/tests/integration_tests/int_tests.template.env b/package/tests/integration_tests/int_tests.template.env deleted file mode 100644 index ab6e322a..00000000 --- a/package/tests/integration_tests/int_tests.template.env +++ /dev/null @@ -1,13 +0,0 @@ -SB_SERVICE_ALIAS1 = "" -SB_SERVICE_ALIAS2 = "" - - -AZUREAPP_TF_INPUTS = "" -GH_TF_PRIVATE_AZUREAPP_URL = "" -GH_TOKEN_DEC = "" -CLP_RESOURSE = "" - -AZURE_SUBSCRIPTION_ID = "" -AZURE_TENANT_ID = "" -AZURE_APPLICATION_ID = "" -AZURE_APPLICATION_KEY_DEC = "" diff --git a/package/tests/integration_tests/int_tests_secrets.template.env b/package/tests/integration_tests/int_tests_secrets.template.env new file mode 100644 index 00000000..5febe50d --- /dev/null +++ b/package/tests/integration_tests/int_tests_secrets.template.env @@ -0,0 +1,6 @@ +AZURE_APPLICATION_KEY_DEC = "" +GH_TOKEN_DEC = "" + +AZURE_SUBSCRIPTION_ID = "" +AZURE_TENANT_ID = "" +AZURE_APPLICATION_ID = "" \ No newline at end of file From f036afaef7961bf0ede55c91ea654c3b34eb92e5 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 22 Oct 2021 13:53:34 -0500 Subject: [PATCH 40/70] A bit of documentation and new KeyVault on Azure --- .../integration_tests/int_test_mock_tf_execute_destroy.py | 6 ++++-- package/tests/integration_tests/int_tests.env | 4 +++- .../tests/integration_tests/int_tests_secrets.template.env | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 50b8eeed..94f0fc18 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -18,12 +18,14 @@ class TestMockTerraformExecuteDestroy(TestCase): - @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') - def setUp(self, patched_api) -> None: + def setUpClass(self): load_dotenv(Path('int_tests.env')) if os.path.isfile(Path('int_tests_secrets.env')): load_dotenv(Path('int_tests_secrets.env')) + @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') + def setUp(self, patched_api) -> None: + self._prepare_mock_api() patched_api.return_value.get_api.return_value = self.mock_api diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env index 76a10261..6289356d 100644 --- a/package/tests/integration_tests/int_tests.env +++ b/package/tests/integration_tests/int_tests.env @@ -1,5 +1,7 @@ +# The following values are used both for local runs and Github CI + SB_SERVICE_ALIAS1 = "ooo1" SB_SERVICE_ALIAS2 = "ooo2" -AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=alexaz-amd-test,KEYVAULT_RG=alexaz-test-amd,SECRET_NAME=test" +AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=CloudShell-Terraform-Shell-ResourceGroup,KEYVAULT_RG=tf-shell-tests,SECRET_NAME=test" GH_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" \ No newline at end of file diff --git a/package/tests/integration_tests/int_tests_secrets.template.env b/package/tests/integration_tests/int_tests_secrets.template.env index 5febe50d..c07ae302 100644 --- a/package/tests/integration_tests/int_tests_secrets.template.env +++ b/package/tests/integration_tests/int_tests_secrets.template.env @@ -1,3 +1,6 @@ +# The following values are used only for local runs and need to be filled to run the integrations tests +# "_DEC" is a shotring and stands for DECrypted + AZURE_APPLICATION_KEY_DEC = "" GH_TOKEN_DEC = "" From 59cacc1e92d540b8e4e956954c71e01bde53f6a6 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 22 Oct 2021 14:06:51 -0500 Subject: [PATCH 41/70] Update int_test_mock_tf_execute_destroy.py --- .../tests/integration_tests/int_test_mock_tf_execute_destroy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 94f0fc18..5768498e 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -17,7 +17,7 @@ class TestMockTerraformExecuteDestroy(TestCase): - + @classmethod def setUpClass(self): load_dotenv(Path('int_tests.env')) if os.path.isfile(Path('int_tests_secrets.env')): From 5578ce70d108d04b4febae6f81fefc66d2b02856 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 22 Oct 2021 15:15:14 -0500 Subject: [PATCH 42/70] Update int_tests.env --- package/tests/integration_tests/int_tests.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env index 6289356d..893a9ce6 100644 --- a/package/tests/integration_tests/int_tests.env +++ b/package/tests/integration_tests/int_tests.env @@ -3,5 +3,5 @@ SB_SERVICE_ALIAS1 = "ooo1" SB_SERVICE_ALIAS2 = "ooo2" -AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=CloudShell-Terraform-Shell-ResourceGroup,KEYVAULT_RG=tf-shell-tests,SECRET_NAME=test" +AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=tf-tests-keyvault,KEYVAULT_RG=cs-tf-tests-rg,SECRET_NAME=test" GH_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" \ No newline at end of file From cba93b1877738eaea22b0ae8efa9c6385a1ef70c Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 22 Oct 2021 15:52:04 -0500 Subject: [PATCH 43/70] fixed the downloader integration test and also testing how 'package.' in imports works on gh --- .../helper_objects/downloader_integration_context.py | 3 +-- .../helper_objects/integration_context.py | 4 ++-- .../helper_services/service_attributes_factory.py | 2 +- package/tests/integration_tests/int_test_downloader.py | 7 ++++--- .../integration_tests/int_test_mock_tf_execute_destroy.py | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/package/tests/integration_tests/helper_objects/downloader_integration_context.py b/package/tests/integration_tests/helper_objects/downloader_integration_context.py index 6620ea4c..d019b71a 100644 --- a/package/tests/integration_tests/helper_objects/downloader_integration_context.py +++ b/package/tests/integration_tests/helper_objects/downloader_integration_context.py @@ -17,8 +17,7 @@ def __init__(self): self._env_vars.cs_server, self._env_vars.cs_user, self._env_vars.cs_pass, - self._env_vars.cs_domain, - os.environ.get("SB_SERVICE_ALIAS") + self._env_vars.cs_domain ) self._set_context() self._logger = get_qs_logger(log_group=self.context.resource.name) diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index 46909858..eb424503 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -6,8 +6,8 @@ from cloudshell.logging.qs_logger import get_qs_logger from cloudshell.shell.core.driver_context import ResourceCommandContext -from tests.integration_tests.helper_objects.env_vars import EnvVars -from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder +from package.tests.integration_tests.helper_objects.env_vars import EnvVars +from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder class IntegrationData(object): diff --git a/package/tests/integration_tests/helper_services/service_attributes_factory.py b/package/tests/integration_tests/helper_services/service_attributes_factory.py index c3f5c5b2..3f7695e7 100644 --- a/package/tests/integration_tests/helper_services/service_attributes_factory.py +++ b/package/tests/integration_tests/helper_services/service_attributes_factory.py @@ -2,7 +2,7 @@ from cloudshell.api.cloudshell_api import NameValuePair -from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES +from package.tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES class ServiceAttributesMockBuilder: diff --git a/package/tests/integration_tests/int_test_downloader.py b/package/tests/integration_tests/int_test_downloader.py index 941fb453..cc507d38 100644 --- a/package/tests/integration_tests/int_test_downloader.py +++ b/package/tests/integration_tests/int_test_downloader.py @@ -6,9 +6,10 @@ from cloudshell.iac.terraform.services.sandbox_messages import SandboxMessagesService from shells.generic_terraform_service.src.data_model import GenericTerraformService from cloudshell.iac.terraform.models.shell_helper import ShellHelperObject -from tests.constants import GH_TF_PUBLIC_HELLO_URL_FILE, GH_TF_PUBLIC_HELLO_URL_FOLDER, TERRAFORM_EXEC_FILE, \ +from package.tests.constants import GH_TF_PUBLIC_HELLO_URL_FILE, GH_TF_PUBLIC_HELLO_URL_FOLDER, TERRAFORM_EXEC_FILE, \ SHELL_NAME, TF_HELLO_FILE -from tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData +from package.tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData + from cloudshell.iac.terraform.tagging.tags import TagsManager from cloudshell.iac.terraform.services.svc_attribute_handler import ServiceAttrHandler @@ -63,7 +64,7 @@ def _test_download_terraform_module(self, url: str, branch: str): def test_public_and_private_hello_dl(self): self._test_download_terraform_module(GH_TF_PUBLIC_HELLO_URL_FILE, "") - self._test_download_terraform_module(os.environ.get("GH_TF_PRIVATE_HELLO_URL"), "") + self._test_download_terraform_module(os.environ.get("GITHUB_TF_PRIVATE_HELLO_URL"), "") self._test_download_terraform_module(GH_TF_PUBLIC_HELLO_URL_FOLDER, "") def test_download_terraform_executable(self): diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 5768498e..b0e2d4a2 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -3,17 +3,17 @@ # from cloudshell.api.cloudshell_api import NameValuePair from cloudshell.api.cloudshell_api import NameValuePair from dotenv import load_dotenv -from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE +from package.tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE from typing import Callable -from tests.integration_tests.helper_objects.integration_context import IntegrationData +from package.tests.integration_tests.helper_objects.integration_context import IntegrationData import os from unittest import TestCase from pathlib import Path -from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder +from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder class TestMockTerraformExecuteDestroy(TestCase): From 6a3ea68bf2ec12a9d8199d7a804fd94792a8f95d Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 22 Oct 2021 15:53:24 -0500 Subject: [PATCH 44/70] Update downloader_integration_context.py --- .../helper_objects/downloader_integration_context.py | 1 - 1 file changed, 1 deletion(-) diff --git a/package/tests/integration_tests/helper_objects/downloader_integration_context.py b/package/tests/integration_tests/helper_objects/downloader_integration_context.py index d019b71a..7443a312 100644 --- a/package/tests/integration_tests/helper_objects/downloader_integration_context.py +++ b/package/tests/integration_tests/helper_objects/downloader_integration_context.py @@ -1,5 +1,4 @@ from unittest import mock -import os from cloudshell.api.cloudshell_api import CloudShellAPISession from cloudshell.logging.qs_logger import get_qs_logger From 68807f9ce66ceeaf103d5bbb492be0e743e25ca3 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 22 Oct 2021 15:57:19 -0500 Subject: [PATCH 45/70] removed the "package." from imports of integration tests --- .../helper_objects/downloader_integration_context.py | 2 +- .../integration_tests/helper_objects/integration_context.py | 4 ++-- .../helper_services/service_attributes_factory.py | 2 +- .../integration_tests/int_test_mock_tf_execute_destroy.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package/tests/integration_tests/helper_objects/downloader_integration_context.py b/package/tests/integration_tests/helper_objects/downloader_integration_context.py index 7443a312..433c5958 100644 --- a/package/tests/integration_tests/helper_objects/downloader_integration_context.py +++ b/package/tests/integration_tests/helper_objects/downloader_integration_context.py @@ -5,7 +5,7 @@ from cloudshell.shell.core.driver_context import ResourceCommandContext -from package.tests.integration_tests.helper_objects.env_vars import EnvVars +from tests.integration_tests.helper_objects.env_vars import EnvVars class IntegrationData(object): diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index eb424503..46909858 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -6,8 +6,8 @@ from cloudshell.logging.qs_logger import get_qs_logger from cloudshell.shell.core.driver_context import ResourceCommandContext -from package.tests.integration_tests.helper_objects.env_vars import EnvVars -from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder +from tests.integration_tests.helper_objects.env_vars import EnvVars +from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder class IntegrationData(object): diff --git a/package/tests/integration_tests/helper_services/service_attributes_factory.py b/package/tests/integration_tests/helper_services/service_attributes_factory.py index 3f7695e7..c3f5c5b2 100644 --- a/package/tests/integration_tests/helper_services/service_attributes_factory.py +++ b/package/tests/integration_tests/helper_services/service_attributes_factory.py @@ -2,7 +2,7 @@ from cloudshell.api.cloudshell_api import NameValuePair -from package.tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES +from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES class ServiceAttributesMockBuilder: diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index b0e2d4a2..7260194c 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -6,14 +6,14 @@ from package.tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE from typing import Callable -from package.tests.integration_tests.helper_objects.integration_context import IntegrationData +from tests.integration_tests.helper_objects.integration_context import IntegrationData import os from unittest import TestCase from pathlib import Path -from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder +from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder class TestMockTerraformExecuteDestroy(TestCase): From 8361665d1244b2b4030dbbf92a78a8b369fcf846 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 22 Oct 2021 15:59:02 -0500 Subject: [PATCH 46/70] Update int_test_mock_tf_execute_destroy.py --- .../integration_tests/int_test_mock_tf_execute_destroy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 7260194c..1aa71141 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -3,8 +3,7 @@ # from cloudshell.api.cloudshell_api import NameValuePair from cloudshell.api.cloudshell_api import NameValuePair from dotenv import load_dotenv -from package.tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE -from typing import Callable +from package.typing import Callable from tests.integration_tests.helper_objects.integration_context import IntegrationData From 129b442e882edf85b52c48459eb0a3db7dd701a1 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 22 Oct 2021 16:02:10 -0500 Subject: [PATCH 47/70] Update int_test_mock_tf_execute_destroy.py --- .../integration_tests/int_test_mock_tf_execute_destroy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py index 1aa71141..5768498e 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py @@ -3,7 +3,8 @@ # from cloudshell.api.cloudshell_api import NameValuePair from cloudshell.api.cloudshell_api import NameValuePair from dotenv import load_dotenv -from package.typing import Callable +from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE +from typing import Callable from tests.integration_tests.helper_objects.integration_context import IntegrationData From 9bac2d1254dd68da7f6483f1a29f567c4dcf4ad4 Mon Sep 17 00:00:00 2001 From: qualidan Date: Fri, 22 Oct 2021 16:29:50 -0500 Subject: [PATCH 48/70] Update downloader_integration_context.py --- .../helper_objects/downloader_integration_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/tests/integration_tests/helper_objects/downloader_integration_context.py b/package/tests/integration_tests/helper_objects/downloader_integration_context.py index 433c5958..7443a312 100644 --- a/package/tests/integration_tests/helper_objects/downloader_integration_context.py +++ b/package/tests/integration_tests/helper_objects/downloader_integration_context.py @@ -5,7 +5,7 @@ from cloudshell.shell.core.driver_context import ResourceCommandContext -from tests.integration_tests.helper_objects.env_vars import EnvVars +from package.tests.integration_tests.helper_objects.env_vars import EnvVars class IntegrationData(object): From 48868b100929a2034f1c2ea365bac2d1df9ffd80 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 27 Oct 2021 13:15:33 -0500 Subject: [PATCH 49/70] multiple changes (Please see description) 1) Refactored Context_data for integrations tests and divided into real and mocked 2) Changed the folder structure revolving real/mock integration tests 3) Added INT_TESTSING_README.ME documentation for integration tests 4) Moved some of the tf modules used for testing to be part of the package --- .../integration_tests/INT-TEST-README.md | 91 +++++++++ .../helper_objects/integration_context.py | 82 ++++++++- package/tests/integration_tests/int_tests.env | 8 +- .../int_tests_secrets.template.env | 2 + .../mock_api_based_tests/__init__.py | 0 .../int_test_mock_tf_execute_destroy.py | 44 ++--- .../real_api_based_tests/__init__.py | 0 .../int_test_real_downloader.py} | 0 .../int_test_real_tf_execute_destroy.py | 77 ++++---- .../tf_modules/azure-paas-mssql/main.tf | 174 ++++++++++++++++++ .../tf_modules/azure-paas-mssql/runSQLCMD.sh | 34 ++++ .../tf_modules/azure-paas-mssql/vars.tf | 69 +++++++ .../tf_modules/azure-vault/main.tf | 39 ++++ .../tf_modules/azure-vault/vars.tf | 9 + 14 files changed, 570 insertions(+), 59 deletions(-) create mode 100644 package/tests/integration_tests/INT-TEST-README.md create mode 100644 package/tests/integration_tests/mock_api_based_tests/__init__.py rename package/tests/integration_tests/{ => mock_api_based_tests}/int_test_mock_tf_execute_destroy.py (88%) create mode 100644 package/tests/integration_tests/real_api_based_tests/__init__.py rename package/tests/integration_tests/{int_test_downloader.py => real_api_based_tests/int_test_real_downloader.py} (100%) rename package/tests/integration_tests/{ => real_api_based_tests}/int_test_real_tf_execute_destroy.py (75%) create mode 100644 package/tests/integration_tests/tf_modules/azure-paas-mssql/main.tf create mode 100644 package/tests/integration_tests/tf_modules/azure-paas-mssql/runSQLCMD.sh create mode 100644 package/tests/integration_tests/tf_modules/azure-paas-mssql/vars.tf create mode 100644 package/tests/integration_tests/tf_modules/azure-vault/main.tf create mode 100644 package/tests/integration_tests/tf_modules/azure-vault/vars.tf diff --git a/package/tests/integration_tests/INT-TEST-README.md b/package/tests/integration_tests/INT-TEST-README.md new file mode 100644 index 00000000..7253aa17 --- /dev/null +++ b/package/tests/integration_tests/INT-TEST-README.md @@ -0,0 +1,91 @@ +# Integration Tests for CloudShell-Terraform-Shell + +The purpose of this documentation section is to provide explanations regarding different aspects of the integrations tests usage and concepts used the making. + +## Content + +1) Structure +2) Helper Objects/Services +3) How to: + 1) Run + 2) Extend + +### Structure + +The Integration tests are devided into two types: +1) Mocked (mock_api_based_tests) +2) Real (real_api_based_tests) + +The Mocked (*1) tests will be using the Mock API (multiple patched functions of the API) during tests +for that it shall also use the MockAPIIntegrationData Object that inherits from BaseIntegrationData part of helper_objects\integration_context.py + +he Real (*2) tests will be using the Real API (Communicate with an actual live CloudShell server) during tests +* for that one would need to create a int_tests_secrets.env file based on the values mentioned in int_tests_secrets.template.env + in order for the test to work. +* In addition, one must also create a blueprint (and a derived sandbox) with two services (which the tests will use for running the tests) +for that it shall also use the RealAPIIntegrationData Object that inherits from BaseIntegrationData part of helper_objects\integration_context.py + +### Helper Objects/Services + +1) Helper Object (helper_objects) + 1) Environment Variables (helper_objects\env_vars.py) + * Used to hold generatl information regarding the Real reservation testing against + 2) Integration Context Data (helper_objects\integration_context.py) + * Used both by "Real" and "Mocked" in order to construct the data for the tests wether real or mocked +2) Helper Services (helper_services) + 1) Serice Attributes Factory (helper_services\service_attributes_factory.py) + * Used by "Mocked" tests to create empty attributes + +### How To + +#### Run +prerequisite: have a a unit tests plugin installed (e.g. unittests) + + 1) As IDEs have sometimes issues with importing packages, best way is to open the project from it`s "Package" as root dir + 2) For running real tests: + 1) Instantiate a CloudShell server + 2) Create a blueprint with two TF Services + 3) create a int_tests_secrets.env file and fill it with the variables (and their values) from int_tests_secrets.template.env + 3) For mock tests: + 1) either from IDE or CLI : python -m unittest discover -p "int_test_mock*.py" + +#### Extend + +Test cases part of int_test_mock_tf_execute_destroy.py have several sections: + +1) Patching (bypassing the use of real api / server requirements): + + For example: + + @patch('cloudshell.iac.terraform.services.tf_proc_exec.TfProcExec.can_destroy_run') + @patch('cloudshell.iac.terraform.terraform_shell.SandboxDataHandler') + @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') + +2) Making sure the patched functions/objects are initialized with the right values: + + For example: + + def test_execute_and_destroy_azure_vault(self, patch_api, patched_sbdata_handler, can_destroy_run): + + can_destroy_run.return_value = True + + patch_api.return_value.get_api.return_value = self.mock_api + + mock_sbdata_handler = Mock() + mock_sbdata_handler.get_tf_working_dir = self._get_mocked_tf_working_dir + mock_sbdata_handler.set_tf_working_dir = self._set_mocked_tf_working_dir + patched_sbdata_handler.return_value = mock_sbdata_handler + +3) the run_execute_and_destroy function that holds 3 parameters + what function needs to run before executing + * create such a function and modify the attributes values there, so it will match the test case requirements here + what function needs to run before destroying + The data used for the test (self.integration_data1) + + For example: + +self.run_execute_and_destroy( + pre_exec_function=self.pre_exec_azure_vault, + pre_destroy_function=self.pre_destroy, + integration_data=self.integration_data1 + ) \ No newline at end of file diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index 46909858..625530b7 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -8,9 +8,87 @@ from tests.integration_tests.helper_objects.env_vars import EnvVars from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder +from abc import ABCMeta -class IntegrationData(object): +class BaseIntegrationData(metaclass=ABCMeta): + def __init__(self, service_name: str, is_api_real: bool = True, mock_api: Mock = None): + self._sb_service_alias = service_name + self._logger = get_qs_logger(log_group=service_name) + + def _set_context(self): + raise NotImplementedError() + + def _build_context(self): + self.context = mock.create_autospec(ResourceCommandContext) + self.context.connectivity = mock.MagicMock() + + def _set_general_context_resource_data(self): + self.context.resource = mock.MagicMock() + self.context.resource.attributes = dict() + self.context.resource.name = self._sb_service_alias + self.context.resource.model = 'Generic Terraform Service' + + def create_tf_shell(self): + self._config = TerraformShellConfig(write_sandbox_messages=True, update_live_status=True) + self.tf_shell = TerraformShell(self.context, self._logger, self._config) + + +class RealAPIIntegrationData(BaseIntegrationData): + def __init__(self, service_name: str): + BaseIntegrationData.__init__(self, service_name) + self._env_vars = EnvVars() + self.api = CloudShellAPISession( + self._env_vars.cs_server, + self._env_vars.cs_user, + self._env_vars.cs_pass, + self._env_vars.cs_domain + ) + self._set_context() + BaseIntegrationData.create_tf_shell(self) + + def _set_context(self): + BaseIntegrationData._build_context(self) + self.context.connectivity.server_address = self._env_vars.cs_server + self.context.connectivity.admin_auth_token = self.api.authentication.xmlrpc_token + BaseIntegrationData._set_general_context_resource_data(self) + self.set_context_resource_attributes_from_cs() + self.context.reservation = mock.MagicMock() + self.context.reservation.reservation_id = self._env_vars.cs_res_id + self.context.reservation.domain = self._env_vars.cs_domain + + def set_context_resource_attributes_from_cs(self, the_only_attribute_to_update: str = ""): + services = self.api.GetReservationDetails(self._env_vars.cs_res_id, disableCache=True) \ + .ReservationDescription.Services + for service in services: + if service.Alias == self._sb_service_alias: + for attribute in service.Attributes: + if the_only_attribute_to_update and attribute.Name == the_only_attribute_to_update: + self.context.resource.attributes[attribute.Name] = attribute.Value + return + elif not the_only_attribute_to_update: + self.context.resource.attributes[attribute.Name] = attribute.Value + + +class MockAPIIntegrationData(BaseIntegrationData): + def __init__(self, service_name: str, mock_api: Mock): + BaseIntegrationData.__init__(self, service_name) + + self.api = mock_api + self.api.authentication.xmlrpc_token = Mock() + + self._set_context() + BaseIntegrationData.create_tf_shell(self) + + def _set_context(self): + BaseIntegrationData._build_context(self) + BaseIntegrationData._set_general_context_resource_data(self) + self.context.resource.attributes = ServiceAttributesMockBuilder.create_empty_attributes() + self.context.reservation = mock.MagicMock() + self.context.reservation.reservation_id = "mock_reservation_id" + + +'''class IntegrationData_(object): def __init__(self, service_name: str, is_api_real: bool = True, mock_api: Mock = None): # self._sb_service_alias = service_name @@ -66,4 +144,4 @@ def set_context_resource_attributes_from_cs(self, the_only_attribute_to_update: def create_tf_shell(self): self._config = TerraformShellConfig(write_sandbox_messages=True, update_live_status=True) - self.tf_shell = TerraformShell(self.context, self._logger, self._config) + self.tf_shell = TerraformShell(self.context, self._logger, self._config)''' diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env index 893a9ce6..88920385 100644 --- a/package/tests/integration_tests/int_tests.env +++ b/package/tests/integration_tests/int_tests.env @@ -1,7 +1,9 @@ # The following values are used both for local runs and Github CI -SB_SERVICE_ALIAS1 = "ooo1" -SB_SERVICE_ALIAS2 = "ooo2" + AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=tf-tests-keyvault,KEYVAULT_RG=cs-tf-tests-rg,SECRET_NAME=test" -GH_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" \ No newline at end of file +GITHUB_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" + + +# GH_TF_PRIVATE_AZUREAPP_URL = "https://github.com/QualiSystemsLab/CloudShell-Terraform-Shell/tree/master/package/tests/integration_tests/terraform_test_modules/azure-vault" \ No newline at end of file diff --git a/package/tests/integration_tests/int_tests_secrets.template.env b/package/tests/integration_tests/int_tests_secrets.template.env index c07ae302..0f0f5f57 100644 --- a/package/tests/integration_tests/int_tests_secrets.template.env +++ b/package/tests/integration_tests/int_tests_secrets.template.env @@ -1,5 +1,7 @@ # The following values are used only for local runs and need to be filled to run the integrations tests # "_DEC" is a shotring and stands for DECrypted +SB_SERVICE_ALIAS1 = "" +SB_SERVICE_ALIAS2 = "" AZURE_APPLICATION_KEY_DEC = "" GH_TOKEN_DEC = "" diff --git a/package/tests/integration_tests/mock_api_based_tests/__init__.py b/package/tests/integration_tests/mock_api_based_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py similarity index 88% rename from package/tests/integration_tests/int_test_mock_tf_execute_destroy.py rename to package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py index 5768498e..2d1ec131 100644 --- a/package/tests/integration_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py @@ -6,7 +6,7 @@ from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE from typing import Callable -from tests.integration_tests.helper_objects.integration_context import IntegrationData +from tests.integration_tests.helper_objects.integration_context import MockAPIIntegrationData import os from unittest import TestCase @@ -19,9 +19,9 @@ class TestMockTerraformExecuteDestroy(TestCase): @classmethod def setUpClass(self): - load_dotenv(Path('int_tests.env')) - if os.path.isfile(Path('int_tests_secrets.env')): - load_dotenv(Path('int_tests_secrets.env')) + load_dotenv(Path('../int_tests.env')) + if os.path.isfile(Path('../int_tests_secrets.env')): + load_dotenv(Path('../int_tests_secrets.env')) @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') def setUp(self, patched_api) -> None: @@ -37,8 +37,8 @@ def setUp(self, patched_api) -> None: self._prepare_integration_data() def _prepare_integration_data(self): - self.integration_data1 = IntegrationData(self._service1.Alias, False, self.mock_api) - self.integration_data2 = IntegrationData(self._service2.Alias, False, self.mock_api) + self.integration_data1 = MockAPIIntegrationData(self._service1.Alias, self.mock_api) + self.integration_data2 = MockAPIIntegrationData(self._service2.Alias, self.mock_api) def _prepare_mock_services(self): self._service1 = Mock() @@ -62,16 +62,16 @@ def _prepare_mock_api(self): '''------------------------------ Generic Execute/Destroy functions ---------------------------------''' - def run_execute(self, pre_exec_function: Callable, integration_data: IntegrationData): + def run_execute(self, pre_exec_function: Callable, integration_data: MockAPIIntegrationData): self.pre_exec_prep(pre_exec_function, integration_data) integration_data.tf_shell.execute_terraform() - def run_destroy(self, pre_destroy_function: Callable, integration_data: IntegrationData): + def run_destroy(self, pre_destroy_function: Callable, integration_data: MockAPIIntegrationData): self.pre_destroy_prep(pre_destroy_function, integration_data) integration_data.tf_shell.destroy_terraform() def run_execute_and_destroy(self, pre_exec_function: Callable, pre_destroy_function: Callable, - integration_data: IntegrationData): + integration_data: MockAPIIntegrationData): self.run_execute(pre_exec_function, integration_data) self.run_destroy(pre_destroy_function, integration_data) @@ -95,10 +95,10 @@ def test_execute_and_destroy_azure_vault(self, patch_api, patched_sbdata_handler '''------------------------------ Functions : general _pre prep functions ---------------------------------''' - def pre_exec_prep(self, pre_exec_function: Callable, integration_data: IntegrationData): + def pre_exec_prep(self, pre_exec_function: Callable, integration_data: MockAPIIntegrationData): pre_exec_function(integration_data) - def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: IntegrationData): + def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: MockAPIIntegrationData): pre_destroy_function(integration_data) def clear_sb_data(self): @@ -106,10 +106,10 @@ def clear_sb_data(self): '''------------------------------ Functions : prep before exec -------------------------------------------''' - def pre_exec(self, integration_data: IntegrationData): + def pre_exec(self, integration_data: MockAPIIntegrationData): pass - def pre_exec_azure_vault(self, integration_data: IntegrationData): + def pre_exec_azure_vault(self, integration_data: MockAPIIntegrationData): self._set_attribute_on_mock_service( f"{SHELL_NAME}.Terraform Inputs", os.environ.get("AZUREAPP_TF_INPUTS"), @@ -117,7 +117,7 @@ def pre_exec_azure_vault(self, integration_data: IntegrationData): ) self._set_attribute_on_mock_service( f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GH_TF_PRIVATE_AZUREAPP_URL"), + os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), integration_data ) self._set_attribute_on_mock_service( @@ -144,7 +144,7 @@ def _prepare_service1_before_execute(self, integration_data): self.mock_api.GetReservationDetails.return_value.ReservationDescription.Services = [service1] integration_data.create_tf_shell() - def pre_exec_azure_mssql(self, integration_data: IntegrationData): + def pre_exec_azure_mssql(self, integration_data: MockAPIIntegrationData): self._set_attribute_on_mock_service( f"{SHELL_NAME}.Terraform Inputs", os.environ.get("AZUREMSSQL_TF_INPUTS"), @@ -162,7 +162,7 @@ def pre_exec_azure_mssql(self, integration_data: IntegrationData): ) self._prepare_service1_before_execute(integration_data) - def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: IntegrationData): + def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: MockAPIIntegrationData): self.pre_exec_azure_vault(integration_data) self._set_attribute_on_mock_service( f"{SHELL_NAME}.Remote State Provider", @@ -170,7 +170,7 @@ def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: In integration_data ) - def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: IntegrationData): + def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: MockAPIIntegrationData): self.pre_exec_azure_vault(integration_data) self._set_attribute_on_mock_service( f"{SHELL_NAME}.Remote State Provider", @@ -178,7 +178,7 @@ def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: In integration_data ) - def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: IntegrationData): + def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: MockAPIIntegrationData): self.pre_exec_azure_vault(integration_data) self._set_attribute_on_mock_service( f"{SHELL_NAME}.Remote State Provider", @@ -186,7 +186,7 @@ def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: integration_data ) - def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: IntegrationData): + def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: MockAPIIntegrationData): self.pre_exec_azure_vault(integration_data) self._set_attribute_on_mock_service( f"{SHELL_NAME}.Remote State Provider", @@ -194,7 +194,7 @@ def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: Integ integration_data ) - def pre_exec_azure_vault_without_remote(self, integration_data: IntegrationData): + def pre_exec_azure_vault_without_remote(self, integration_data: MockAPIIntegrationData): self.pre_exec_azure_vault(integration_data) self._set_attribute_on_mock_service( f"{SHELL_NAME}.Remote State Provider", @@ -204,7 +204,7 @@ def pre_exec_azure_vault_without_remote(self, integration_data: IntegrationData) '''------------------------------ Functions : prep before destroy -----------------------------------------''' - def pre_destroy(self, integration_data: IntegrationData): + def pre_destroy(self, integration_data: MockAPIIntegrationData): # As UUID has been created and SB data now contains UUID and Status we must update context so destroy can run for attribute in integration_data.context.resource.attributes: if attribute.Name == f"{SHELL_NAME}.UUID": @@ -212,7 +212,7 @@ def pre_destroy(self, integration_data: IntegrationData): '''------------------------------ Helper Functions ---------------------------------------------------------''' @staticmethod - def _set_attribute_on_mock_service(attr_name: str, attr_value: str, integration_data: IntegrationData): + def _set_attribute_on_mock_service(attr_name: str, attr_value: str, integration_data: MockAPIIntegrationData): for attribute in integration_data.context.resource.attributes: if attribute.Name == attr_name: attribute.Value = attr_value diff --git a/package/tests/integration_tests/real_api_based_tests/__init__.py b/package/tests/integration_tests/real_api_based_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/package/tests/integration_tests/int_test_downloader.py b/package/tests/integration_tests/real_api_based_tests/int_test_real_downloader.py similarity index 100% rename from package/tests/integration_tests/int_test_downloader.py rename to package/tests/integration_tests/real_api_based_tests/int_test_real_downloader.py diff --git a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py b/package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py similarity index 75% rename from package/tests/integration_tests/int_test_real_tf_execute_destroy.py rename to package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py index b62e01b3..e7af2eab 100644 --- a/package/tests/integration_tests/int_test_real_tf_execute_destroy.py +++ b/package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py @@ -2,33 +2,40 @@ from dotenv import load_dotenv from tests.integration_tests.constants import SHELL_NAME from typing import Callable +from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER -from tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData +from tests.integration_tests.helper_objects.integration_context import RealAPIIntegrationData import os +from pathlib import Path from unittest import TestCase class TestRealTerraformExecuteDestroy(TestCase): + @classmethod + def setUpClass(self): + load_dotenv(Path('../int_tests.env')) + if os.path.isfile(Path('../int_tests_secrets.env')): + load_dotenv(Path('../int_tests_secrets.env')) + def setUp(self) -> None: - load_dotenv() - self.integration_data1 = IntegrationData(os.environ.get("SB_SERVICE_ALIAS1"), is_api_real=True) - self.integration_data2 = IntegrationData(os.environ.get("SB_SERVICE_ALIAS2"), is_api_real=True) + self.integration_data1 = RealAPIIntegrationData(os.environ.get("SB_SERVICE_ALIAS1")) + self.integration_data2 = RealAPIIntegrationData(os.environ.get("SB_SERVICE_ALIAS2")) def run_execute_and_destroy( self, pre_exec_function: Callable, pre_destroy_function: Callable, - integration_data: IntegrationData + integration_data: RealAPIIntegrationData ): self.clear_sb_data() self.run_execute(pre_exec_function, integration_data) self.run_destroy(pre_destroy_function, integration_data) - def run_execute(self, pre_exec_function: Callable, integration_data: IntegrationData): + def run_execute(self, pre_exec_function: Callable, integration_data: RealAPIIntegrationData): self.pre_exec_prep(pre_exec_function, integration_data) integration_data.tf_shell.execute_terraform() - def run_destroy(self, pre_destroy_function: Callable, integration_data: IntegrationData): + def run_destroy(self, pre_destroy_function: Callable, integration_data: RealAPIIntegrationData): self.pre_destroy_prep(pre_destroy_function, integration_data) integration_data.tf_shell.destroy_terraform() @@ -57,6 +64,7 @@ def test_execute_dual_mssql(self): try: self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data2) except Exception as e: + print("Error") pass self.run_destroy(pre_destroy_function=self.pre_destroy, integration_data=self.integration_data1) self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data2) @@ -104,10 +112,10 @@ def test_execute_and_destroy_azure_vault_without_remote(self): '''------------------------------ Functions : general _pre prep functions ---------------------------------''' - def pre_exec_prep(self, pre_exec_function: Callable, integration_data: IntegrationData): + def pre_exec_prep(self, pre_exec_function: Callable, integration_data: RealAPIIntegrationData): pre_exec_function(integration_data) - def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: IntegrationData): + def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: RealAPIIntegrationData): pre_destroy_function(integration_data) def clear_sb_data(self): @@ -115,18 +123,23 @@ def clear_sb_data(self): '''------------------------------ Functions : prep before exec -------------------------------------------''' - def pre_exec(self, integration_data: IntegrationData): + def pre_exec(self, integration_data: RealAPIIntegrationData): pass - def pre_exec_azure_vault(self, integration_data: IntegrationData): + def pre_exec_azure_vault(self, integration_data: RealAPIIntegrationData): self._set_attribute_on_service( - f"{SHELL_NAME}.Terraform Inputs", + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", os.environ.get("AZUREAPP_TF_INPUTS"), integration_data ) self._set_attribute_on_service( - f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GH_TF_PRIVATE_AZUREAPP_URL"), + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", + os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), + integration_data + ) + self._set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", + INT_TEST_TF_VER, integration_data ) self._set_attribute_on_service( @@ -135,72 +148,72 @@ def pre_exec_azure_vault(self, integration_data: IntegrationData): integration_data ) - def pre_exec_azure_mssql(self, integration_data: IntegrationData): + def pre_exec_azure_mssql(self, integration_data: RealAPIIntegrationData): self._set_attribute_on_service( - f"{SHELL_NAME}.Terraform Inputs", + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", os.environ.get("AZUREMSSQL_TF_INPUTS"), integration_data ) self._set_attribute_on_service( - f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GH_TF_PRIVATE_AZUREMSSQL_URL"), + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", + os.environ.get("GITHUB_TF_PRIVATE_AZUREMSSQL_URL"), integration_data ) self._set_attribute_on_service( - f"{SHELL_NAME}.UUID", + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}", os.environ.get(""), integration_data ) - def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: IntegrationData): + def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: RealAPIIntegrationData): self.pre_exec_azure_vault(integration_data) self._set_attribute_on_service( - f"{SHELL_NAME}.Remote State Provider", + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", os.environ.get("REMOTE_STATE_PROVIDER_ACCESS_KEY"), integration_data ) - def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: IntegrationData): + def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: RealAPIIntegrationData): self.pre_exec_azure_vault(integration_data) self._set_attribute_on_service( - f"{SHELL_NAME}.Remote State Provider", + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", os.environ.get("REMOTE_STATE_PROVIDER_CLOUD_CRED"), integration_data ) - def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: IntegrationData): + def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: RealAPIIntegrationData): self.pre_exec_azure_vault(integration_data) self._set_attribute_on_service( - f"{SHELL_NAME}.Remote State Provider", + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", os.environ.get("REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE"), integration_data ) - def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: IntegrationData): + def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: RealAPIIntegrationData): self.pre_exec_azure_vault(integration_data) self._set_attribute_on_service( - f"{SHELL_NAME}.Remote State Provider", + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", os.environ.get("REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE"), integration_data ) - def pre_exec_azure_vault_without_remote(self, integration_data: IntegrationData): + def pre_exec_azure_vault_without_remote(self, integration_data: RealAPIIntegrationData): self.pre_exec_azure_vault(integration_data) self._set_attribute_on_service( - f"{SHELL_NAME}.Remote State Provider", + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", os.environ.get(""), integration_data ) '''------------------------------ Functions : prep before destroy -----------------------------------------''' - def pre_destroy(self, integration_data: IntegrationData): + def pre_destroy(self, integration_data: RealAPIIntegrationData): # As UUID has been created and SB data now contains UUID and Status we must update context so destroy can run - integration_data.set_context_resource_attributes_from_cs(f"{SHELL_NAME}.UUID") + integration_data.set_context_resource_attributes_from_cs(f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}") '''------------------------------ Helper Functions ---------------------------------------------------------''' - def _set_attribute_on_service(self, attr_name: str, attr_value: str, integration_data: IntegrationData): + def _set_attribute_on_service(self, attr_name: str, attr_value: str, integration_data: RealAPIIntegrationData): attr_req = [AttributeNameValue(attr_name, attr_value)] integration_data.api.SetServiceAttributesValues( integration_data.context.reservation.reservation_id, diff --git a/package/tests/integration_tests/tf_modules/azure-paas-mssql/main.tf b/package/tests/integration_tests/tf_modules/azure-paas-mssql/main.tf new file mode 100644 index 00000000..191b7367 --- /dev/null +++ b/package/tests/integration_tests/tf_modules/azure-paas-mssql/main.tf @@ -0,0 +1,174 @@ +terraform { + required_providers { + azurerm = { + version = "2.61" + } + } +} + +provider azurerm { + features {} +} + +locals { + sizeMap = { + "small" = "GP_Gen5_2", + "medium" = "GP_GEN5_4", + "large" = "GP_GEN5_8", + "extra-large" = "GP_GEN5_16", + "s2" = "S2", + "s3" = "S3", + "s4" = "S4" + } + + mgmtServer1 = "165.204.84.17" + mgmtServer2 = "165.204.88.17" + listOfIps = var.ALLOWED_IPS + ipList = compact(split(",", var.ALLOWED_IPS)) + + sanitizedRG = replace(var.RESOURCE_GROUP_NAME, "_", "") #Server Name cannot contain anything other than letters, numbers, and hyphens + serverName = lower("${var.SERVER_NAME}-${local.sanitizedRG}") + + customTags = { + "APPID" = var.APP_ID, + "APPLICATION OWNER" = var.APP_OWNER, + "APPLICATION NAME" = var.APP_NAME + } +} + +resource "azurerm_mssql_server" "default" { + name = local.serverName + resource_group_name = var.RESOURCE_GROUP_NAME + location = var.LOCATION + version = "12.0" + public_network_access_enabled = true + administrator_login = var.SERVER_USERNAME + administrator_login_password = var.SERVER_PASSWORD + + azuread_administrator { + login_username = "ITDBA Admin" + object_id = "ed6204e1-a9c4-408c-bb5e-bcc0d076b3bd" + tenant_id = "3dd8961f-e488-4e60-8e11-a82d994e183d" + } +} + +resource "azurerm_sql_firewall_rule" "ip_list" { + for_each = { for x in local.ipList : x => x } + name = "${each.key}-Requested-IP-Allowance" + resource_group_name = var.RESOURCE_GROUP_NAME + server_name = azurerm_mssql_server.default.name + start_ip_address = trimspace(each.key) + end_ip_address = trimspace(each.key) + depends_on = [ azurerm_mssql_server.default ] +} + +resource "azurerm_sql_firewall_rule" "allow_controlserver_one" { + name = "Allow-Control-Server" + resource_group_name = var.RESOURCE_GROUP_NAME + server_name = azurerm_mssql_server.default.name + start_ip_address = local.mgmtServer1 + end_ip_address = local.mgmtServer1 + depends_on = [ azurerm_mssql_server.default ] +} + +resource "azurerm_sql_firewall_rule" "allow_controlserver_two" { + name = "Allow-ControlServer-2" + resource_group_name = var.RESOURCE_GROUP_NAME + server_name = azurerm_mssql_server.default.name + start_ip_address = local.mgmtServer2 + end_ip_address = local.mgmtServer2 + depends_on = [ azurerm_mssql_server.default ] +} + +resource "azurerm_sql_firewall_rule" "allow_azure_services" { + name = "AllowAzureServices" + resource_group_name = var.RESOURCE_GROUP_NAME + server_name = azurerm_mssql_server.default.name + start_ip_address = "0.0.0.0" + end_ip_address = "0.0.0.0" + depends_on = [ azurerm_mssql_server.default ] +} + +resource "azurerm_mssql_database" "default" { + name = var.DB_NAME + server_id = azurerm_mssql_server.default.id + sku_name = lookup(local.sizeMap, lower(var.DB_SIZE), "GP_Gen5_2") + create_mode = "Default" + collation = var.COLLATION + max_size_gb = var.DB_STORAGE + depends_on = [ azurerm_mssql_server.default ] + + short_term_retention_policy { + retention_days = 35 + } + + tags = merge(local.customTags) +} + +/* Commented out while we are deciding on proper password handling procedures +resource "random_password" "RO_PASSWORD" { + length = 24 + special = true + override_special = "_%@" + min_special = 1 + min_lower = 1 + min_upper = 1 + min_numeric = 1 +} + +resource "random_password" "RW_PASSWORD" { + length = 24 + special = true + override_special = "_%@" + min_special = 1 + min_lower = 1 + min_upper = 1 + min_numeric = 1 +} + +resource "random_password" "WO_PASSWORD" { + length = 24 + special = true + override_special = "_%@" + min_special = 1 + min_lower = 1 + min_upper = 1 + min_numeric = 1 +} +*/ + +# resource "null_resource" "addUsers" { +# depends_on = [ azurerm_mssql_database.default ] + +# provisioner "local-exec" { +# on_failure = fail +# command = "chmod +x runSQLCMD.sh && ./runSQLCMD.sh ${var.DB_PASSWORD} ${var.RO_PASSWORD} ${var.RW_PASSWORD}" +# interpreter = ["/bin/bash", "-c"] + +# environment = { +# FQDN = azurerm_mssql_server.default.fully_qualified_domain_name +# SERVER_USERNAME = var.SERVER_USERNAME +# SERVER_PASSWORD = var.SERVER_PASSWORD +# DB_NAME = var.DB_NAME +# DB_USER = var.DB_USERNAME +# // DB_PASSWORD = var.DB_PASSWORD +# RO_USER = "${var.APP_NAME}_RO" +# RW_USER = "${var.APP_NAME}_RW" +# O_USER = "${var.APP_NAME}_O" +# RO_PASS = var.RO_PASSWORD +# RW_PASS = var.RW_PASSWORD +# O_PASS = var.O_PASSWORD +# } +# } +# } + + +output "DB_HOSTNAME" { + value = azurerm_mssql_server.default.fully_qualified_domain_name +} +output "DB_USER" { + value = var.DB_USERNAME +} +output "DB_NAME" { + value = var.DB_NAME +} diff --git a/package/tests/integration_tests/tf_modules/azure-paas-mssql/runSQLCMD.sh b/package/tests/integration_tests/tf_modules/azure-paas-mssql/runSQLCMD.sh new file mode 100644 index 00000000..3c7f21a5 --- /dev/null +++ b/package/tests/integration_tests/tf_modules/azure-paas-mssql/runSQLCMD.sh @@ -0,0 +1,34 @@ +apt install gnupg -y + +curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +curl https://packages.microsoft.com/config/debian/10/prod.list | tee /etc/apt/sources.list.d/msprod.list + + +apt update + +export ACCEPT_EULA=y +export DEBIAN_FRONTEND=noninteractive +apt-get install -y --no-install-recommends mssql-tools unixodbc-dev + +export PATH=$PATH:/opt/mssql-tools/bin + +#We create the query like this because sqlcmd on Debian 10 does not support -v +cat << EOF > addUsers.sql +CREATE USER [$O_USER] WITH PASSWORD = "$O_PASS" , DEFAULT_SCHEMA = dbo; +ALTER ROLE db_owner ADD MEMBER [$O_USER]; +GO + +CREATE USER [$RO_USER] WITH PASSWORD = "$RO_PASS" , DEFAULT_SCHEMA = dbo; +ALTER ROLE db_datareader ADD MEMBER [$RO_USER]; +GO + +CREATE USER [$RW_USER] WITH PASSWORD = "$RW_PASS" , DEFAULT_SCHEMA = dbo; +ALTER ROLE db_datawriter ADD MEMBER [$RW_USER]; +GO + +CREATE USER [$DB_USER] WITH PASSWORD = "$DB_PASS" , DEFAULT_SCHEMA = dbo; +ALTER ROLE db_owner ADD MEMBER [$DB_USER] +GO +EOF + +sqlcmd -C -S $FQDN -d $DB_NAME -U $SERVER_USERNAME -P $SERVER_PASSWORD -i addUsers.sql \ No newline at end of file diff --git a/package/tests/integration_tests/tf_modules/azure-paas-mssql/vars.tf b/package/tests/integration_tests/tf_modules/azure-paas-mssql/vars.tf new file mode 100644 index 00000000..6e65ea76 --- /dev/null +++ b/package/tests/integration_tests/tf_modules/azure-paas-mssql/vars.tf @@ -0,0 +1,69 @@ +variable "RESOURCE_GROUP_NAME" { + type = string +} +variable "LOCATION" { + type = string +} +variable "SERVER_NAME" { + type = string +} +variable "SERVER_USERNAME" { +} +variable "SERVER_PASSWORD" { + sensitive = true +} +variable "DB_NAME" { +} +variable "DB_USERNAME" { +} +variable "DB_PASSWORD" { + sensitive = true +} + +variable "DB_SIZE" { + description = "Desired Service Level - small (2 v-core) , medium (4), or large (8)" + default = "small" + type = string + + validation { + condition = contains(["small", "medium", "large", "s3", "s4", "s2"], lower(var.DB_SIZE)) + error_message = "Valid Values: small, medium, large." + } +} + +variable "DB_STORAGE" { + type = number + default = 50 +} + +variable "ALLOWED_IPS" { + description = "List of IPs to be allowed, comma-separated. Example: 192.168.1.1,192.168.1.20,192.168.1.222" + type = string +} + +variable "COLLATION" { + default = "SQL_LATIN1_GENERAL_CP1_CI_AS" + type = string +} + +variable "RO_PASSWORD" { + type = string + sensitive = true +} +variable "RW_PASSWORD" { + type = string + sensitive = true +} +variable "O_PASSWORD" { + type = string + sensitive = true +} + +#TAGS +variable "APP_ID" { +} +variable "APP_OWNER" { +} +variable "APP_NAME" { +} + diff --git a/package/tests/integration_tests/tf_modules/azure-vault/main.tf b/package/tests/integration_tests/tf_modules/azure-vault/main.tf new file mode 100644 index 00000000..d9dd2556 --- /dev/null +++ b/package/tests/integration_tests/tf_modules/azure-vault/main.tf @@ -0,0 +1,39 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "=2.46.0" + } + } +} + +provider "azurerm" { + features {} +} + +data "azurerm_key_vault" "keyvault" { + name = var.KEYVAULT_NAME + resource_group_name = var.KEYVAULT_RG +} + +data "azurerm_key_vault_secret" "keyvault_get" { + name = var.SECRET_NAME + key_vault_id = data.azurerm_key_vault.keyvault.id +} + +output "SECRET_VALUE"{ + value = data.azurerm_key_vault_secret.keyvault_get.value + sensitive = true +} + +output "SECRET_VALUE_2" { + value = "my_secret" + sensitive = true +} + +output "BLA1" { + value = "bla1" +} +output "BLA2" { + value = "bla2" +} \ No newline at end of file diff --git a/package/tests/integration_tests/tf_modules/azure-vault/vars.tf b/package/tests/integration_tests/tf_modules/azure-vault/vars.tf new file mode 100644 index 00000000..5114dc22 --- /dev/null +++ b/package/tests/integration_tests/tf_modules/azure-vault/vars.tf @@ -0,0 +1,9 @@ +variable KEYVAULT_NAME { + +} +variable KEYVAULT_RG { + +} +variable SECRET_NAME { + +} From fc8705f9e9b7284acccae5df0bb36566213f9535 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 27 Oct 2021 13:30:09 -0500 Subject: [PATCH 50/70] Update int_test_real_tf_execute_destroy.py --- .../real_api_based_tests/int_test_real_tf_execute_destroy.py | 1 - 1 file changed, 1 deletion(-) diff --git a/package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py b/package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py index e7af2eab..d145bbe5 100644 --- a/package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py +++ b/package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py @@ -1,6 +1,5 @@ from cloudshell.api.cloudshell_api import AttributeNameValue from dotenv import load_dotenv -from tests.integration_tests.constants import SHELL_NAME from typing import Callable from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER From 65ac3a48cb0e9b1201664d11ffa825c677924d4e Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 27 Oct 2021 13:55:11 -0500 Subject: [PATCH 51/70] Update int_tests.env --- package/tests/integration_tests/int_tests.env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env index 88920385..d1936f78 100644 --- a/package/tests/integration_tests/int_tests.env +++ b/package/tests/integration_tests/int_tests.env @@ -1,6 +1,7 @@ # The following values are used both for local runs and Github CI - +SB_SERVICE_ALIAS1 = "ALIAS1" +SB_SERVICE_ALIAS2 = "ALIAS2" AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=tf-tests-keyvault,KEYVAULT_RG=cs-tf-tests-rg,SECRET_NAME=test" GITHUB_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" From 1f6709ed83de5d60a08848d2765b4c59787364b6 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 27 Oct 2021 14:58:01 -0500 Subject: [PATCH 52/70] no need for mock alias var --- package/tests/integration_tests/int_tests.env | 3 --- .../mock_api_based_tests/int_test_mock_tf_execute_destroy.py | 5 +++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env index d1936f78..12199147 100644 --- a/package/tests/integration_tests/int_tests.env +++ b/package/tests/integration_tests/int_tests.env @@ -1,8 +1,5 @@ # The following values are used both for local runs and Github CI -SB_SERVICE_ALIAS1 = "ALIAS1" -SB_SERVICE_ALIAS2 = "ALIAS2" - AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=tf-tests-keyvault,KEYVAULT_RG=cs-tf-tests-rg,SECRET_NAME=test" GITHUB_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" diff --git a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py index 2d1ec131..b36ab0e6 100644 --- a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py @@ -19,6 +19,7 @@ class TestMockTerraformExecuteDestroy(TestCase): @classmethod def setUpClass(self): + print(os.getcwd()) load_dotenv(Path('../int_tests.env')) if os.path.isfile(Path('../int_tests_secrets.env')): load_dotenv(Path('../int_tests_secrets.env')) @@ -42,10 +43,10 @@ def _prepare_integration_data(self): def _prepare_mock_services(self): self._service1 = Mock() - self._service1.Alias = os.environ.get("SB_SERVICE_ALIAS1") + self._service1.Alias = "MOCK_ALIAS1" self._service1.Attributes = ServiceAttributesMockBuilder.create_empty_attributes() self._service2 = Mock() - self._service2.Alias = os.environ.get("SB_SERVICE_ALIAS2") + self._service2.Alias = "MOCK_ALIAS2" self._service2.Attributes = ServiceAttributesMockBuilder.create_empty_attributes() def _prepare_mock_api(self): From ed6a6f2ac2eabea556849dd66fd5333844f9e048 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 27 Oct 2021 15:04:44 -0500 Subject: [PATCH 53/70] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e64ef9e..3a35cc13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,6 @@ jobs: run: | pwd - cd package/tests/integration_tests + cd package/tests/integration_tests/mock_api_based_tests pwd python -m unittest discover -p "int_test_mock*.py" From a9368f3763905a481c6efdd405d8e9911c1f3670 Mon Sep 17 00:00:00 2001 From: alexazarh Date: Fri, 5 Nov 2021 12:52:48 -0500 Subject: [PATCH 54/70] clean ups --- package/tests/integration_tests/constants.py | 1 - .../helper_objects/integration_context.py | 94 ++++--------------- .../int_tests_secrets.template.env | 2 +- .../int_test_mock_tf_execute_destroy.py | 37 ++++++-- .../tf_modules/azure-paas-mssql/main.tf | 88 ----------------- .../tf_modules/azure-paas-mssql/outputs.tf | 9 ++ .../tf_modules/azure-paas-mssql/runSQLCMD.sh | 34 ------- .../tf_modules/azure-vault/main.tf | 17 ---- .../tf_modules/azure-vault/outputs.tf | 16 ++++ 9 files changed, 70 insertions(+), 228 deletions(-) create mode 100644 package/tests/integration_tests/tf_modules/azure-paas-mssql/outputs.tf delete mode 100644 package/tests/integration_tests/tf_modules/azure-paas-mssql/runSQLCMD.sh create mode 100644 package/tests/integration_tests/tf_modules/azure-vault/outputs.tf diff --git a/package/tests/integration_tests/constants.py b/package/tests/integration_tests/constants.py index 88fb902f..be8b9c6c 100644 --- a/package/tests/integration_tests/constants.py +++ b/package/tests/integration_tests/constants.py @@ -1,5 +1,4 @@ SHELL_NAME = "Generic Terraform Service" -UUID_ATTRIBUTE = f"{SHELL_NAME}.UUID" INT_TEST_TF_VER = "0.15.1" INT_TEST_CLP_RESOURSE = "real_azure" diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index 625530b7..d8078171 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -8,35 +8,40 @@ from tests.integration_tests.helper_objects.env_vars import EnvVars from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder -from abc import ABCMeta +from abc import ABCMeta, abstractmethod class BaseIntegrationData(metaclass=ABCMeta): - def __init__(self, service_name: str, is_api_real: bool = True, mock_api: Mock = None): + def __init__(self, service_name: str): self._sb_service_alias = service_name self._logger = get_qs_logger(log_group=service_name) + self._build_context() + self._set_general_context_resource_data() + self._set_context() + self.create_tf_shell() + + @abstractmethod def _set_context(self): raise NotImplementedError() def _build_context(self): self.context = mock.create_autospec(ResourceCommandContext) - self.context.connectivity = mock.MagicMock() + self.context.connectivity = Mock() def _set_general_context_resource_data(self): - self.context.resource = mock.MagicMock() + self.context.resource = Mock() self.context.resource.attributes = dict() self.context.resource.name = self._sb_service_alias self.context.resource.model = 'Generic Terraform Service' def create_tf_shell(self): - self._config = TerraformShellConfig(write_sandbox_messages=True, update_live_status=True) - self.tf_shell = TerraformShell(self.context, self._logger, self._config) + _config = TerraformShellConfig(write_sandbox_messages=True, update_live_status=True) + self.tf_shell = TerraformShell(self.context, self._logger, _config) class RealAPIIntegrationData(BaseIntegrationData): def __init__(self, service_name: str): - BaseIntegrationData.__init__(self, service_name) self._env_vars = EnvVars() self.api = CloudShellAPISession( self._env_vars.cs_server, @@ -44,16 +49,13 @@ def __init__(self, service_name: str): self._env_vars.cs_pass, self._env_vars.cs_domain ) - self._set_context() - BaseIntegrationData.create_tf_shell(self) + BaseIntegrationData.__init__(self, service_name) def _set_context(self): - BaseIntegrationData._build_context(self) self.context.connectivity.server_address = self._env_vars.cs_server self.context.connectivity.admin_auth_token = self.api.authentication.xmlrpc_token - BaseIntegrationData._set_general_context_resource_data(self) self.set_context_resource_attributes_from_cs() - self.context.reservation = mock.MagicMock() + self.context.reservation = Mock() self.context.reservation.reservation_id = self._env_vars.cs_res_id self.context.reservation.domain = self._env_vars.cs_domain @@ -72,76 +74,12 @@ def set_context_resource_attributes_from_cs(self, the_only_attribute_to_update: class MockAPIIntegrationData(BaseIntegrationData): def __init__(self, service_name: str, mock_api: Mock): - BaseIntegrationData.__init__(self, service_name) - self.api = mock_api self.api.authentication.xmlrpc_token = Mock() - self._set_context() - BaseIntegrationData.create_tf_shell(self) + BaseIntegrationData.__init__(self, service_name) def _set_context(self): - BaseIntegrationData._build_context(self) - BaseIntegrationData._set_general_context_resource_data(self) self.context.resource.attributes = ServiceAttributesMockBuilder.create_empty_attributes() - self.context.reservation = mock.MagicMock() + self.context.reservation = Mock() self.context.reservation.reservation_id = "mock_reservation_id" - - -'''class IntegrationData_(object): - def __init__(self, service_name: str, is_api_real: bool = True, mock_api: Mock = None): - # - self._sb_service_alias = service_name - - if is_api_real: - self._env_vars = EnvVars() - self.api = CloudShellAPISession( - self._env_vars.cs_server, - self._env_vars.cs_user, - self._env_vars.cs_pass, - self._env_vars.cs_domain - ) - - else: - self.api = mock_api - self.api.authentication.xmlrpc_token = Mock() - self._set_context(is_api_real) - self._logger = get_qs_logger(log_group=self.context.resource.name) - self.create_tf_shell() - - def _set_context(self, is_api_real: bool): - self.context = mock.create_autospec(ResourceCommandContext) - self.context.connectivity = mock.MagicMock() - if is_api_real: - self.context.connectivity.server_address = self._env_vars.cs_server - self.context.connectivity.admin_auth_token = self.api.authentication.xmlrpc_token - - self.context.resource = mock.MagicMock() - self.context.resource.attributes = dict() - self.context.resource.name = self._sb_service_alias - self.context.resource.model = 'Generic Terraform Service' - if is_api_real: - self.set_context_resource_attributes_from_cs() - self.context.reservation = mock.MagicMock() - self.context.reservation.reservation_id = self._env_vars.cs_res_id - self.context.reservation.domain = self._env_vars.cs_domain - else: - self.context.resource.attributes = ServiceAttributesMockBuilder.create_empty_attributes() - self.context.reservation = mock.MagicMock() - self.context.reservation.reservation_id = "mock_reservation_id" - - def set_context_resource_attributes_from_cs(self, the_only_attribute_to_update: str = ""): - services = self.api.GetReservationDetails(self._env_vars.cs_res_id, disableCache=True) \ - .ReservationDescription.Services - for service in services: - if service.Alias == self._sb_service_alias: - for attribute in service.Attributes: - if the_only_attribute_to_update and attribute.Name == the_only_attribute_to_update: - self.context.resource.attributes[attribute.Name] = attribute.Value - return - elif not the_only_attribute_to_update: - self.context.resource.attributes[attribute.Name] = attribute.Value - - def create_tf_shell(self): - self._config = TerraformShellConfig(write_sandbox_messages=True, update_live_status=True) - self.tf_shell = TerraformShell(self.context, self._logger, self._config)''' diff --git a/package/tests/integration_tests/int_tests_secrets.template.env b/package/tests/integration_tests/int_tests_secrets.template.env index 0f0f5f57..100be69c 100644 --- a/package/tests/integration_tests/int_tests_secrets.template.env +++ b/package/tests/integration_tests/int_tests_secrets.template.env @@ -1,5 +1,5 @@ # The following values are used only for local runs and need to be filled to run the integrations tests -# "_DEC" is a shotring and stands for DECrypted +# "_DEC" is a short for DECrypted SB_SERVICE_ALIAS1 = "" SB_SERVICE_ALIAS2 = "" diff --git a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py index b36ab0e6..fd2c2dc0 100644 --- a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py @@ -17,6 +17,8 @@ class TestMockTerraformExecuteDestroy(TestCase): + # region Test Setup + @classmethod def setUpClass(self): print(os.getcwd()) @@ -51,7 +53,7 @@ def _prepare_mock_services(self): def _prepare_mock_api(self): self.mock_api = Mock() - self.mock_api.DecryptPassword = _decrypt_password + self.mock_api.DecryptPassword = self._decrypt_password self.mock_api.GetResourceDetails.return_value.ResourceFamilyName = 'Cloud Provider' self.mock_api.GetResourceDetails.return_value.ResourceModelName = 'Microsoft Azure' self.mock_api.GetResourceDetails.return_value.ResourceAttributes = [ @@ -61,7 +63,9 @@ def _prepare_mock_api(self): NameValuePair(Name="Azure Application Key", Value=os.environ.get("AZURE_APPLICATION_KEY_DEC")) ] - '''------------------------------ Generic Execute/Destroy functions ---------------------------------''' + # endregion + + # region Generic Helper Execute/Destroy functions def run_execute(self, pre_exec_function: Callable, integration_data: MockAPIIntegrationData): self.pre_exec_prep(pre_exec_function, integration_data) @@ -76,29 +80,40 @@ def run_execute_and_destroy(self, pre_exec_function: Callable, pre_destroy_funct self.run_execute(pre_exec_function, integration_data) self.run_destroy(pre_destroy_function, integration_data) - '''------------------------------ Test Cases ---------------------------------''' + # endregion + + # region Test Cases @patch('cloudshell.iac.terraform.services.tf_proc_exec.TfProcExec.can_destroy_run') @patch('cloudshell.iac.terraform.terraform_shell.SandboxDataHandler') @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') def test_execute_and_destroy_azure_vault(self, patch_api, patched_sbdata_handler, can_destroy_run): + # arrange can_destroy_run.return_value = True patch_api.return_value.get_api.return_value = self.mock_api mock_sbdata_handler = Mock() mock_sbdata_handler.get_tf_working_dir = self._get_mocked_tf_working_dir mock_sbdata_handler.set_tf_working_dir = self._set_mocked_tf_working_dir patched_sbdata_handler.return_value = mock_sbdata_handler + # act + # todo - split execture and destroy to add asserts after each self.run_execute_and_destroy( pre_exec_function=self.pre_exec_azure_vault, pre_destroy_function=self.pre_destroy, integration_data=self.integration_data1 ) + # assert + # todo - add assert + + # endregion '''------------------------------ Functions : general _pre prep functions ---------------------------------''' + # todo - remove this layer def pre_exec_prep(self, pre_exec_function: Callable, integration_data: MockAPIIntegrationData): pre_exec_function(integration_data) + # todo - remove this layer def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: MockAPIIntegrationData): pre_destroy_function(integration_data) @@ -107,6 +122,7 @@ def clear_sb_data(self): '''------------------------------ Functions : prep before exec -------------------------------------------''' + # todo - merge with real api and move to helper class def pre_exec(self, integration_data: MockAPIIntegrationData): pass @@ -211,7 +227,9 @@ def pre_destroy(self, integration_data: MockAPIIntegrationData): if attribute.Name == f"{SHELL_NAME}.UUID": attribute.Value = integration_data.tf_shell._tf_service.attributes[f"{SHELL_NAME}.UUID"] - '''------------------------------ Helper Functions ---------------------------------------------------------''' + # region Helper Functions + #todo - check if makes sense to move to a helper class + @staticmethod def _set_attribute_on_mock_service(attr_name: str, attr_value: str, integration_data: MockAPIIntegrationData): for attribute in integration_data.context.resource.attributes: @@ -225,9 +243,10 @@ def _get_mocked_tf_working_dir(self): def _set_mocked_tf_working_dir(self, tf_working_dir: str): self._mocked_tf_working_dir = tf_working_dir + @staticmethod + def _decrypt_password(x): + result = Mock() + result.Value = x + return result -def _decrypt_password(x): - # result = mock.MagicMock() - result = Mock() - result.Value = x - return result + # endregion diff --git a/package/tests/integration_tests/tf_modules/azure-paas-mssql/main.tf b/package/tests/integration_tests/tf_modules/azure-paas-mssql/main.tf index 191b7367..d69a8b27 100644 --- a/package/tests/integration_tests/tf_modules/azure-paas-mssql/main.tf +++ b/package/tests/integration_tests/tf_modules/azure-paas-mssql/main.tf @@ -21,8 +21,6 @@ locals { "s4" = "S4" } - mgmtServer1 = "165.204.84.17" - mgmtServer2 = "165.204.88.17" listOfIps = var.ALLOWED_IPS ipList = compact(split(",", var.ALLOWED_IPS)) @@ -62,24 +60,6 @@ resource "azurerm_sql_firewall_rule" "ip_list" { depends_on = [ azurerm_mssql_server.default ] } -resource "azurerm_sql_firewall_rule" "allow_controlserver_one" { - name = "Allow-Control-Server" - resource_group_name = var.RESOURCE_GROUP_NAME - server_name = azurerm_mssql_server.default.name - start_ip_address = local.mgmtServer1 - end_ip_address = local.mgmtServer1 - depends_on = [ azurerm_mssql_server.default ] -} - -resource "azurerm_sql_firewall_rule" "allow_controlserver_two" { - name = "Allow-ControlServer-2" - resource_group_name = var.RESOURCE_GROUP_NAME - server_name = azurerm_mssql_server.default.name - start_ip_address = local.mgmtServer2 - end_ip_address = local.mgmtServer2 - depends_on = [ azurerm_mssql_server.default ] -} - resource "azurerm_sql_firewall_rule" "allow_azure_services" { name = "AllowAzureServices" resource_group_name = var.RESOURCE_GROUP_NAME @@ -104,71 +84,3 @@ resource "azurerm_mssql_database" "default" { tags = merge(local.customTags) } - -/* Commented out while we are deciding on proper password handling procedures -resource "random_password" "RO_PASSWORD" { - length = 24 - special = true - override_special = "_%@" - min_special = 1 - min_lower = 1 - min_upper = 1 - min_numeric = 1 -} - -resource "random_password" "RW_PASSWORD" { - length = 24 - special = true - override_special = "_%@" - min_special = 1 - min_lower = 1 - min_upper = 1 - min_numeric = 1 -} - -resource "random_password" "WO_PASSWORD" { - length = 24 - special = true - override_special = "_%@" - min_special = 1 - min_lower = 1 - min_upper = 1 - min_numeric = 1 -} -*/ - -# resource "null_resource" "addUsers" { -# depends_on = [ azurerm_mssql_database.default ] - -# provisioner "local-exec" { -# on_failure = fail -# command = "chmod +x runSQLCMD.sh && ./runSQLCMD.sh ${var.DB_PASSWORD} ${var.RO_PASSWORD} ${var.RW_PASSWORD}" -# interpreter = ["/bin/bash", "-c"] - -# environment = { -# FQDN = azurerm_mssql_server.default.fully_qualified_domain_name -# SERVER_USERNAME = var.SERVER_USERNAME -# SERVER_PASSWORD = var.SERVER_PASSWORD -# DB_NAME = var.DB_NAME -# DB_USER = var.DB_USERNAME -# // DB_PASSWORD = var.DB_PASSWORD -# RO_USER = "${var.APP_NAME}_RO" -# RW_USER = "${var.APP_NAME}_RW" -# O_USER = "${var.APP_NAME}_O" -# RO_PASS = var.RO_PASSWORD -# RW_PASS = var.RW_PASSWORD -# O_PASS = var.O_PASSWORD -# } -# } -# } - - -output "DB_HOSTNAME" { - value = azurerm_mssql_server.default.fully_qualified_domain_name -} -output "DB_USER" { - value = var.DB_USERNAME -} -output "DB_NAME" { - value = var.DB_NAME -} diff --git a/package/tests/integration_tests/tf_modules/azure-paas-mssql/outputs.tf b/package/tests/integration_tests/tf_modules/azure-paas-mssql/outputs.tf new file mode 100644 index 00000000..de1d2053 --- /dev/null +++ b/package/tests/integration_tests/tf_modules/azure-paas-mssql/outputs.tf @@ -0,0 +1,9 @@ +output "DB_HOSTNAME" { + value = azurerm_mssql_server.default.fully_qualified_domain_name +} +output "DB_USER" { + value = var.DB_USERNAME +} +output "DB_NAME" { + value = var.DB_NAME +} diff --git a/package/tests/integration_tests/tf_modules/azure-paas-mssql/runSQLCMD.sh b/package/tests/integration_tests/tf_modules/azure-paas-mssql/runSQLCMD.sh deleted file mode 100644 index 3c7f21a5..00000000 --- a/package/tests/integration_tests/tf_modules/azure-paas-mssql/runSQLCMD.sh +++ /dev/null @@ -1,34 +0,0 @@ -apt install gnupg -y - -curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - -curl https://packages.microsoft.com/config/debian/10/prod.list | tee /etc/apt/sources.list.d/msprod.list - - -apt update - -export ACCEPT_EULA=y -export DEBIAN_FRONTEND=noninteractive -apt-get install -y --no-install-recommends mssql-tools unixodbc-dev - -export PATH=$PATH:/opt/mssql-tools/bin - -#We create the query like this because sqlcmd on Debian 10 does not support -v -cat << EOF > addUsers.sql -CREATE USER [$O_USER] WITH PASSWORD = "$O_PASS" , DEFAULT_SCHEMA = dbo; -ALTER ROLE db_owner ADD MEMBER [$O_USER]; -GO - -CREATE USER [$RO_USER] WITH PASSWORD = "$RO_PASS" , DEFAULT_SCHEMA = dbo; -ALTER ROLE db_datareader ADD MEMBER [$RO_USER]; -GO - -CREATE USER [$RW_USER] WITH PASSWORD = "$RW_PASS" , DEFAULT_SCHEMA = dbo; -ALTER ROLE db_datawriter ADD MEMBER [$RW_USER]; -GO - -CREATE USER [$DB_USER] WITH PASSWORD = "$DB_PASS" , DEFAULT_SCHEMA = dbo; -ALTER ROLE db_owner ADD MEMBER [$DB_USER] -GO -EOF - -sqlcmd -C -S $FQDN -d $DB_NAME -U $SERVER_USERNAME -P $SERVER_PASSWORD -i addUsers.sql \ No newline at end of file diff --git a/package/tests/integration_tests/tf_modules/azure-vault/main.tf b/package/tests/integration_tests/tf_modules/azure-vault/main.tf index d9dd2556..17b5a05f 100644 --- a/package/tests/integration_tests/tf_modules/azure-vault/main.tf +++ b/package/tests/integration_tests/tf_modules/azure-vault/main.tf @@ -20,20 +20,3 @@ data "azurerm_key_vault_secret" "keyvault_get" { name = var.SECRET_NAME key_vault_id = data.azurerm_key_vault.keyvault.id } - -output "SECRET_VALUE"{ - value = data.azurerm_key_vault_secret.keyvault_get.value - sensitive = true -} - -output "SECRET_VALUE_2" { - value = "my_secret" - sensitive = true -} - -output "BLA1" { - value = "bla1" -} -output "BLA2" { - value = "bla2" -} \ No newline at end of file diff --git a/package/tests/integration_tests/tf_modules/azure-vault/outputs.tf b/package/tests/integration_tests/tf_modules/azure-vault/outputs.tf new file mode 100644 index 00000000..6530e4b1 --- /dev/null +++ b/package/tests/integration_tests/tf_modules/azure-vault/outputs.tf @@ -0,0 +1,16 @@ +output "SECRET_VALUE"{ + value = data.azurerm_key_vault_secret.keyvault_get.value + sensitive = true +} + +### mock outputs for testing +output "SECRET_VALUE_2" { + value = "my_secret" + sensitive = true +} +output "BLA1" { + value = "bla1" +} +output "BLA2" { + value = "bla2" +} \ No newline at end of file From 37cfed21493f25347778647caacdb9480097b4d3 Mon Sep 17 00:00:00 2001 From: alexazarh Date: Fri, 5 Nov 2021 12:57:19 -0500 Subject: [PATCH 55/70] fixing lint --- .../mock_api_based_tests/int_test_mock_tf_execute_destroy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py index fd2c2dc0..defe7622 100644 --- a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py @@ -228,7 +228,7 @@ def pre_destroy(self, integration_data: MockAPIIntegrationData): attribute.Value = integration_data.tf_shell._tf_service.attributes[f"{SHELL_NAME}.UUID"] # region Helper Functions - #todo - check if makes sense to move to a helper class + # todo - check if makes sense to move to a helper class @staticmethod def _set_attribute_on_mock_service(attr_name: str, attr_value: str, integration_data: MockAPIIntegrationData): From 368d595d3f863533ab97bc80ba7a5ca179573d3e Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 13:18:11 -0600 Subject: [PATCH 56/70] changes to Integration tests --- .github/workflows/ci.yml | 1 + .../terraform/services/clp_envvar_handler.py | 2 +- .../iac/terraform/terraform_shell.py | 3 + package/test_requirements.txt | 3 + package/tests/constants.py | 5 + package/tests/integration_tests/constants.py | 4 + .../downloader_integration_context.py | 47 ---- .../helper_objects/integration_context.py | 9 +- .../helper_objects/mock_tests_data.py | 71 +++++ .../helper_objects/real_tests_data.py | 16 ++ .../helper_services/azure_validator.py | 20 ++ .../helper_services/mock_tests_prepper.py | 128 +++++++++ .../helper_services/real_tests_prepper.py | 100 +++++++ .../service_attributes_factory.py | 3 +- package/tests/integration_tests/int_tests.env | 9 + .../int_test_mock_downloader.py | 113 ++++++++ .../int_test_mock_tf_execute_destroy.py | 239 +++------------- .../int_test_real_downloader.py | 39 ++- .../int_test_real_tf_execute_destroy.py | 258 +++++------------- .../tf_modules/azure-paas-mssql/main.tf | 0 .../tf_modules/azure-paas-mssql/runSQLCMD.sh | 0 .../tf_modules/azure-paas-mssql/vars.tf | 0 .../tf_modules/azure-vault/main.tf | 0 .../tf_modules/azure-vault/vars.tf | 0 24 files changed, 617 insertions(+), 453 deletions(-) delete mode 100644 package/tests/integration_tests/helper_objects/downloader_integration_context.py create mode 100644 package/tests/integration_tests/helper_objects/mock_tests_data.py create mode 100644 package/tests/integration_tests/helper_objects/real_tests_data.py create mode 100644 package/tests/integration_tests/helper_services/azure_validator.py create mode 100644 package/tests/integration_tests/helper_services/mock_tests_prepper.py create mode 100644 package/tests/integration_tests/helper_services/real_tests_prepper.py create mode 100644 package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py rename package/tests/{integration_tests => tests_helper_files}/tf_modules/azure-paas-mssql/main.tf (100%) rename package/tests/{integration_tests => tests_helper_files}/tf_modules/azure-paas-mssql/runSQLCMD.sh (100%) rename package/tests/{integration_tests => tests_helper_files}/tf_modules/azure-paas-mssql/vars.tf (100%) rename package/tests/{integration_tests => tests_helper_files}/tf_modules/azure-vault/main.tf (100%) rename package/tests/{integration_tests => tests_helper_files}/tf_modules/azure-vault/vars.tf (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a35cc13..c94bde10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,7 @@ jobs: AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} + CURRENT_BRANCH = ${GITHUB_REF##*/} run: | pwd diff --git a/package/cloudshell/iac/terraform/services/clp_envvar_handler.py b/package/cloudshell/iac/terraform/services/clp_envvar_handler.py index 51505dce..bbd6f065 100644 --- a/package/cloudshell/iac/terraform/services/clp_envvar_handler.py +++ b/package/cloudshell/iac/terraform/services/clp_envvar_handler.py @@ -13,7 +13,7 @@ def set_env_vars_based_on_clp(self): @staticmethod def does_attribute_match(clp_res_model, clp_attribute, attr_name_to_check) -> bool: - if f"{clp_res_model}.{clp_attribute.Name}" == attr_name_to_check or clp_attribute.Name == attr_name_to_check: + if f"{clp_res_model}.{attr_name_to_check}" == clp_attribute.Name or attr_name_to_check == clp_attribute.Name: return True return False diff --git a/package/cloudshell/iac/terraform/terraform_shell.py b/package/cloudshell/iac/terraform/terraform_shell.py index bbf7b36f..a71141b3 100644 --- a/package/cloudshell/iac/terraform/terraform_shell.py +++ b/package/cloudshell/iac/terraform/terraform_shell.py @@ -85,6 +85,9 @@ def _validate_remote_backend_or_existing_working_dir(self, sandbox_data_handler, not LocalDir.does_working_dir_exists(sandbox_data_handler.get_tf_working_dir()): self._handle_error_output(shell_helper, f"Missing local folder {sandbox_data_handler.get_tf_working_dir()}") + def get_tf_service(self): + return self._tf_service + @staticmethod def _destroy_passed(sandbox_data_handler): return sandbox_data_handler.get_status(DESTROY_STATUS) == DESTROY_PASSED diff --git a/package/test_requirements.txt b/package/test_requirements.txt index dace1ea8..e81be25f 100644 --- a/package/test_requirements.txt +++ b/package/test_requirements.txt @@ -1,3 +1,6 @@ mock==4.0.3 python-dotenv==0.18.0 pathlib==1.0.1 + +azure-common==1.1.27 +azure-mgmt==5.0.0 \ No newline at end of file diff --git a/package/tests/constants.py b/package/tests/constants.py index 44960afa..b65b82cd 100644 --- a/package/tests/constants.py +++ b/package/tests/constants.py @@ -8,3 +8,8 @@ TERRAFORM_EXEC_FILE = "terraform.exe" TF_HELLO_FILE = "hello.tf" SHELL_NAME = "Generic Terraform Service" + +MOCK_ALIAS1 = "MOCK_ALIAS1" +MOCK_ALIAS2 = "MOCK_ALIAS2" + + diff --git a/package/tests/integration_tests/constants.py b/package/tests/integration_tests/constants.py index 88fb902f..69d2bc30 100644 --- a/package/tests/integration_tests/constants.py +++ b/package/tests/integration_tests/constants.py @@ -20,3 +20,7 @@ class ATTRIBUTE_NAMES: BRANCH = "Branch" CLOUD_PROVIDER = "Cloud Provider" UUID = "UUID" + +EXPECTED_VAULT_TF_OUTPUTS = "BLA1=bla1,BLA2=bla2" +EXPECTED_VAULT_TF_SENSETIVE_OUTPUTS_DEC = "SECRET_VALUE=test,SECRET_VALUE_2=my_secret" +EXPECTED_VAULT_TF_SENSETIVE_OUTPUTS_ENC = 'nwH0Qhxnn7ntDpWxsvol+89EPFFSvYE16gHjACzsQRDzvXXnNFc4WfYpgq5w1kw8' \ No newline at end of file diff --git a/package/tests/integration_tests/helper_objects/downloader_integration_context.py b/package/tests/integration_tests/helper_objects/downloader_integration_context.py deleted file mode 100644 index 7443a312..00000000 --- a/package/tests/integration_tests/helper_objects/downloader_integration_context.py +++ /dev/null @@ -1,47 +0,0 @@ -from unittest import mock - -from cloudshell.api.cloudshell_api import CloudShellAPISession -from cloudshell.logging.qs_logger import get_qs_logger -from cloudshell.shell.core.driver_context import ResourceCommandContext - - -from package.tests.integration_tests.helper_objects.env_vars import EnvVars - - -class IntegrationData(object): - def __init__(self): - - self._env_vars = EnvVars() - self.real_api = CloudShellAPISession( - self._env_vars.cs_server, - self._env_vars.cs_user, - self._env_vars.cs_pass, - self._env_vars.cs_domain - ) - self._set_context() - self._logger = get_qs_logger(log_group=self.context.resource.name) - - # self._create_driver() - - def _set_context(self): - self.context = mock.create_autospec(ResourceCommandContext) - self.context.connectivity = mock.MagicMock() - self.context.connectivity.server_address = self._env_vars.cs_server - self.context.connectivity.admin_auth_token = self.real_api.authentication.xmlrpc_token - - self.context.resource = mock.MagicMock() - self.context.resource.attributes = dict() - self.context.resource.name = self._env_vars.sb_service_alias - self.context.resource.model = 'Generic Terraform Service' - self.set_context_resource_attributes() - - self.context.reservation = mock.MagicMock() - self.context.reservation.reservation_id = self._env_vars.cs_res_id - self.context.reservation.domain = self._env_vars.cs_domain - - def set_context_resource_attributes(self): - services = self.real_api.GetReservationDetails(self._env_vars.cs_res_id).ReservationDescription.Services - for service in services: - if service.Alias == self._env_vars.sb_service_alias: - for attribute in service.Attributes: - self.context.resource.attributes[attribute.Name] = attribute.Value diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index 625530b7..e2ba0271 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -6,13 +6,16 @@ from cloudshell.logging.qs_logger import get_qs_logger from cloudshell.shell.core.driver_context import ResourceCommandContext -from tests.integration_tests.helper_objects.env_vars import EnvVars -from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder +# from tests.integration_tests.helper_objects.env_vars import EnvVars +from package.tests.integration_tests.helper_objects.env_vars import EnvVars +# from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder +from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder + from abc import ABCMeta class BaseIntegrationData(metaclass=ABCMeta): - def __init__(self, service_name: str, is_api_real: bool = True, mock_api: Mock = None): + def __init__(self, service_name: str): self._sb_service_alias = service_name self._logger = get_qs_logger(log_group=service_name) diff --git a/package/tests/integration_tests/helper_objects/mock_tests_data.py b/package/tests/integration_tests/helper_objects/mock_tests_data.py new file mode 100644 index 00000000..e4ef9851 --- /dev/null +++ b/package/tests/integration_tests/helper_objects/mock_tests_data.py @@ -0,0 +1,71 @@ +import os +from unittest.mock import Mock + +from package.tests.integration_tests.helper_objects.integration_context import MockAPIIntegrationData +from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder + +from cloudshell.api.cloudshell_api import NameValuePair, AttributeNameValue + + +class MockTestsData(object): + def __init__(self): + self.mock_api = Mock() + self.service1 = Mock() + self.service2 = Mock() + self._mocked_tf_working_dir = '' + self.service1.Alias = '' + self.service1.Attributes = None + self.service2.Alias = '' + self.service2.Attributes = None + + self._prepare_mock_services() + self._prepare_mock_api() + + def prepare_integration_data(self): + self.integration_data1 = MockAPIIntegrationData(self.service1.Alias, self.mock_api) + self.integration_data2 = MockAPIIntegrationData(self.service2.Alias, self.mock_api) + + def _prepare_mock_services(self): + self.service1 = Mock() + self.service1.Alias = "MOCK_ALIAS1" + self.service1.Attributes = ServiceAttributesMockBuilder.create_empty_attributes() + self.service2 = Mock() + self.service2.Alias = "MOCK_ALIAS2" + self.service2.Attributes = ServiceAttributesMockBuilder.create_empty_attributes() + + def _prepare_mock_api(self): + self.mock_api = Mock() + self.mock_api.DecryptPassword = _decrypt_password + self.mock_api.GetReservationDetails.return_value.ReservationDescription.Services = [self.service1, + self.service2] + + # self.mock_api.SetServiceAttributesValues = self._set_service_attributes_values + + self.mock_api.GetResourceDetails.return_value.ResourceFamilyName = 'Cloud Provider' + self.mock_api.GetResourceDetails.return_value.ResourceModelName = 'Microsoft Azure' + self.mock_api.GetResourceDetails.return_value.ResourceAttributes = [ + NameValuePair(Name="Azure Subscription ID", Value=os.environ.get("AZURE_SUBSCRIPTION_ID")), + NameValuePair(Name="Azure Tenant ID", Value=os.environ.get("AZURE_TENANT_ID")), + NameValuePair(Name="Azure Application ID", Value=os.environ.get("AZURE_APPLICATION_ID")), + NameValuePair(Name="Azure Application Key", Value=os.environ.get("AZURE_APPLICATION_KEY_DEC")) + ] + + def _get_mocked_tf_working_dir(self): + return self._mocked_tf_working_dir + + def _set_mocked_tf_working_dir(self, tf_working_dir: str): + self._mocked_tf_working_dir = tf_working_dir + + ''' + @staticmethod + def _set_service_attributes_values(sandbox_id: str, tf_service_name: str, + attr_update_req: list[AttributeNameValue]): + print("") + pass + ''' + +def _decrypt_password(x): + result = Mock() + result.Value = x + return result + diff --git a/package/tests/integration_tests/helper_objects/real_tests_data.py b/package/tests/integration_tests/helper_objects/real_tests_data.py new file mode 100644 index 00000000..685e8e04 --- /dev/null +++ b/package/tests/integration_tests/helper_objects/real_tests_data.py @@ -0,0 +1,16 @@ +import os +from unittest.mock import Mock + +from package.tests.integration_tests.helper_objects.integration_context import MockAPIIntegrationData, \ + RealAPIIntegrationData +from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder + +from cloudshell.api.cloudshell_api import NameValuePair + +class RealTestsData(object): + def __init__(self): + self.integration_data1 = RealAPIIntegrationData(os.environ.get("SB_SERVICE_ALIAS1")) + self.integration_data2 = RealAPIIntegrationData(os.environ.get("SB_SERVICE_ALIAS2")) + + def clear_sb_data(self): + self.integration_data1.api.ClearSandboxData(self.integration_data1.context.reservation.reservation_id) \ No newline at end of file diff --git a/package/tests/integration_tests/helper_services/azure_validator.py b/package/tests/integration_tests/helper_services/azure_validator.py new file mode 100644 index 00000000..cf28e7c8 --- /dev/null +++ b/package/tests/integration_tests/helper_services/azure_validator.py @@ -0,0 +1,20 @@ +import os +from azure.common.credentials import ServicePrincipalCredentials +from azure.mgmt.resource import ResourceManagementClient + + +class AzureValidator(object): + def __init__(self): + self._subscription_id = os.environ['AZURE_SUBSCRIPTION_ID'] + self._client_id = os.environ['AZURE_APPLICATION_ID'] + self._secret = os.environ['AZURE_APPLICATION_KEY_DEC'] + self._tenant = os.environ['AZURE_TENANT_ID'] + self._credentials = ServicePrincipalCredentials( + client_id=self._client_id, + secret=self._secret, + tenant=self._tenant + ) + self._client = ResourceManagementClient(self._credentials, self._subscription_id) + + for item in self._client.resource_groups.list(): + print(item) \ No newline at end of file diff --git a/package/tests/integration_tests/helper_services/mock_tests_prepper.py b/package/tests/integration_tests/helper_services/mock_tests_prepper.py new file mode 100644 index 00000000..9ae35ce2 --- /dev/null +++ b/package/tests/integration_tests/helper_services/mock_tests_prepper.py @@ -0,0 +1,128 @@ +import os +from unittest.mock import Mock + +from package.tests.integration_tests.helper_objects.integration_context import MockAPIIntegrationData +from package.tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, \ + INT_TEST_CLP_RESOURSE +from package.tests.integration_tests.helper_objects.mock_tests_data import MockTestsData + + +def pre_exec_azure_vault(mock_tests_data: MockTestsData): + set_attribute_on_mock_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", + os.environ.get("AZUREAPP_TF_INPUTS"), + mock_tests_data.integration_data1 + ) + set_attribute_on_mock_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", + os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), + mock_tests_data.integration_data1 + ) + set_attribute_on_mock_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TOKEN}", + os.environ.get("GH_TOKEN_DEC"), + mock_tests_data.integration_data1 + ) + set_attribute_on_mock_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", + INT_TEST_TF_VER, + mock_tests_data.integration_data1 + ) + set_attribute_on_mock_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.CLOUD_PROVIDER}", + INT_TEST_CLP_RESOURSE, + mock_tests_data.integration_data1 + ) + prepare_service1_before_execute(mock_tests_data, mock_tests_data.integration_data1) + + +def pre_exec_azure_mssql(mock_tests_data: MockTestsData, + integration_data: MockAPIIntegrationData): + set_attribute_on_mock_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", + os.environ.get("AZUREMSSQL_TF_INPUTS"), + integration_data + ) + set_attribute_on_mock_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", + os.environ.get("GH_TF_PRIVATE_AZUREMSSQL_URL"), + integration_data + ) + set_attribute_on_mock_service( + f"{SHELL_NAME}.UUID", + os.environ.get(""), + integration_data + ) + prepare_service1_before_execute(mock_tests_data, mock_tests_data.integration_data1) + + +def pre_exec_azure_vault_with_remote_access_key_based(mock_tests_data: MockTestsData, + integration_data: MockAPIIntegrationData): + pre_exec_azure_vault(mock_tests_data) + set_attribute_on_mock_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_ACCESS_KEY"), + integration_data + ) + + +def pre_exec_azure_vault_with_remote_cloud_cred_based(mock_tests_data: MockTestsData, + integration_data: MockAPIIntegrationData): + pre_exec_azure_vault(mock_tests_data) + set_attribute_on_mock_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_CLOUD_CRED"), + integration_data + ) + + +def pre_exec_azure_vault_with_remote_invalid_nonexistent(mock_tests_data: MockTestsData, + integration_data: MockAPIIntegrationData): + pre_exec_azure_vault(mock_tests_data) + set_attribute_on_mock_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE"), + integration_data + ) + + +def pre_exec_azure_vault_with_remote_invalid_wrong(mock_tests_data: MockTestsData, + integration_data: MockAPIIntegrationData): + pre_exec_azure_vault(mock_tests_data) + set_attribute_on_mock_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get("REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE"), + integration_data + ) + + +def pre_exec_azure_vault_without_remote(mock_tests_data: MockTestsData, + integration_data: MockAPIIntegrationData): + pre_exec_azure_vault(mock_tests_data) + set_attribute_on_mock_service( + f"{SHELL_NAME}.Remote State Provider", + os.environ.get(""), + integration_data + ) + + +def pre_destroy(integration_data: MockAPIIntegrationData): + # As UUID has been created and SB data now contains UUID and Status we must update context so destroy can run + for attribute in integration_data.context.resource.attributes: + if attribute.Name == f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}": + attribute.Value = integration_data.tf_shell._tf_service.attributes[f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}"] + + +def prepare_service1_before_execute(mock_tests_data: MockTestsData, integration_data): + service1 = Mock() + service1.Alias = integration_data.context.resource.name + service1.Attributes = integration_data.context.resource.attributes + mock_tests_data.mock_api.GetReservationDetails.return_value.ReservationDescription.Services = [service1] + integration_data.create_tf_shell() + + +def set_attribute_on_mock_service(attr_name: str, attr_value: str, integration_data: MockAPIIntegrationData): + for attribute in integration_data.context.resource.attributes: + if attribute.Name == attr_name: + attribute.Value = attr_value + return diff --git a/package/tests/integration_tests/helper_services/real_tests_prepper.py b/package/tests/integration_tests/helper_services/real_tests_prepper.py new file mode 100644 index 00000000..205c1acf --- /dev/null +++ b/package/tests/integration_tests/helper_services/real_tests_prepper.py @@ -0,0 +1,100 @@ +import os + +from cloudshell.api.cloudshell_api import AttributeNameValue + +from package.tests.integration_tests.helper_objects.integration_context import RealAPIIntegrationData +from package.tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER + + +def pre_exec_azure_vault(integration_data: RealAPIIntegrationData): + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", + os.environ.get("AZUREAPP_TF_INPUTS"), + integration_data + ) + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", + os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), + integration_data + ) + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", + INT_TEST_TF_VER, + integration_data + ) + set_attribute_on_service( + f"{SHELL_NAME}.UUID", + os.environ.get(""), + integration_data + ) + + +def pre_exec_azure_vault_with_remote_access_key_based(integration_data: RealAPIIntegrationData): + pre_exec_azure_vault(integration_data) + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", + os.environ.get("REMOTE_STATE_PROVIDER_ACCESS_KEY"), + integration_data + ) + + +def pre_exec_azure_vault_with_remote_cloud_cred_based(integration_data: RealAPIIntegrationData): + pre_exec_azure_vault(integration_data) + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", + os.environ.get("REMOTE_STATE_PROVIDER_CLOUD_CRED"), + integration_data + ) + + +def pre_exec_azure_vault_with_remote_invalid_wrong(integration_data: RealAPIIntegrationData): + pre_exec_azure_vault(integration_data) + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", + os.environ.get("REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE"), + integration_data + ) + + +def pre_exec_azure_vault_with_remote_invalid_nonexistent(integration_data: RealAPIIntegrationData): + pre_exec_azure_vault(integration_data) + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", + os.environ.get("REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE"), + integration_data + ) + + +def pre_exec_azure_mssql(integration_data: RealAPIIntegrationData): + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", + os.environ.get("AZUREMSSQL_TF_INPUTS"), + integration_data + ) + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", + os.environ.get("GITHUB_TF_PRIVATE_AZUREMSSQL_URL"), + integration_data + ) + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}", + os.environ.get(""), + integration_data + ) + + +def post_vault_cleanup(integration_data: RealAPIIntegrationData): + set_attribute_on_service( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", + "", + integration_data + ) + + +def set_attribute_on_service(attr_name: str, attr_value: str, integration_data: RealAPIIntegrationData): + attr_req = [AttributeNameValue(attr_name, attr_value)] + integration_data.api.SetServiceAttributesValues( + integration_data.context.reservation.reservation_id, + integration_data.context.resource.name, + attr_req + ) diff --git a/package/tests/integration_tests/helper_services/service_attributes_factory.py b/package/tests/integration_tests/helper_services/service_attributes_factory.py index c3f5c5b2..7ce57dca 100644 --- a/package/tests/integration_tests/helper_services/service_attributes_factory.py +++ b/package/tests/integration_tests/helper_services/service_attributes_factory.py @@ -2,7 +2,8 @@ from cloudshell.api.cloudshell_api import NameValuePair -from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES +# from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES +from package.tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES class ServiceAttributesMockBuilder: diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env index 12199147..cee0d145 100644 --- a/package/tests/integration_tests/int_tests.env +++ b/package/tests/integration_tests/int_tests.env @@ -3,5 +3,14 @@ AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=tf-tests-keyvault,KEYVAULT_RG=cs-tf-tests-rg,SECRET_NAME=test" GITHUB_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" +REMOTE_STATE_PROVIDER_ACCESS_KEY = "azure_tf_backend1" +REMOTE_STATE_PROVIDER_CLOUD_CRED = "azure_tf_backend2" +REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE = "azure_tf_invalid" +REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE = "mock_azure" + +# CurrentBranch id enviornment variable checkin Github documentaion to implement below +# if enviornment varialbe doesnt exist use a default (do this when loading the .env) +#GITHUB_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/{CURRENTBRANCH2}/terraform/azure-vault" + # GH_TF_PRIVATE_AZUREAPP_URL = "https://github.com/QualiSystemsLab/CloudShell-Terraform-Shell/tree/master/package/tests/integration_tests/terraform_test_modules/azure-vault" \ No newline at end of file diff --git a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py new file mode 100644 index 00000000..25d9f649 --- /dev/null +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py @@ -0,0 +1,113 @@ +import os +from unittest import TestCase +from pathlib import Path + +from dotenv import load_dotenv + +from unittest.mock import patch, Mock + +from cloudshell.iac.terraform.downloaders.downloader import Downloader +from cloudshell.iac.terraform.services.live_status_updater import LiveStatusUpdater +from cloudshell.iac.terraform.services.sandbox_messages import SandboxMessagesService +from shells.generic_terraform_service.src.data_model import GenericTerraformService +from cloudshell.iac.terraform.models.shell_helper import ShellHelperObject +from package.tests.constants import GH_TF_PUBLIC_HELLO_URL_FILE, GH_TF_PUBLIC_HELLO_URL_FOLDER, TERRAFORM_EXEC_FILE, \ + SHELL_NAME, TF_HELLO_FILE, MOCK_ALIAS2, MOCK_ALIAS1 +# from package.tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData +from package.tests.integration_tests.helper_objects.integration_context import MockAPIIntegrationData + +from cloudshell.iac.terraform.tagging.tags import TagsManager +from cloudshell.iac.terraform.services.svc_attribute_handler import ServiceAttrHandler +from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder + + +class TestTerraformDownloader(TestCase): + @classmethod + def setUpClass(self): + load_dotenv(Path('../int_tests.env')) + if os.path.isfile(Path('../int_tests_secrets.env')): + load_dotenv(Path('../int_tests_secrets.env')) + + @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') + def setUp(self, patched_api) -> None: + self._prepare_mock_api() + + self._mocked_tf_working_dir = '' + self._prepare_mock_services() + self.mock_api.GetReservationDetails.return_value.ReservationDescription.Services = [self._service1, + self._service2] + self._prepare_integration_data() + + + + service_resource = GenericTerraformService.create_from_context(self.integration_data1.context) + + sandbox_messages = SandboxMessagesService( + self.integration_data1.api, + self.integration_data1.context.reservation.reservation_id, + self.integration_data1.context.resource.name, + False + ) + + live_status_updater = LiveStatusUpdater( + self.integration_data1.api, + self.integration_data1.context.reservation.reservation_id, + False + ) + + default_tags = TagsManager(self.integration_data1.context.reservation) + + attr_handler = ServiceAttrHandler(service_resource) + + self._driver_helper = ShellHelperObject( + self.integration_data1.api, + self.integration_data1.context.reservation.reservation_id, + service_resource, + self.integration_data1._logger, + sandbox_messages, + live_status_updater, + attr_handler, + default_tags + ) + + def _prepare_integration_data(self): + self.integration_data1 = MockAPIIntegrationData(self._service1.Alias, self.mock_api) + self.integration_data2 = MockAPIIntegrationData(self._service2.Alias, self.mock_api) + + def _prepare_mock_api(self): + self.mock_api = Mock() + + def _prepare_mock_services(self): + self._service1 = Mock() + self._service1.Alias = MOCK_ALIAS1 + self._service1.Attributes = ServiceAttributesMockBuilder.create_empty_attributes() + self._service2 = Mock() + self._service2.Alias = MOCK_ALIAS2 + self._service2.Attributes = ServiceAttributesMockBuilder.create_empty_attributes() + + def _test_download_terraform_module(self, url: str, branch: str): + self.integration_data1.context.resource.attributes[ + f"{SHELL_NAME}.Github Terraform Module URL"] = url + self._driver_helper.tf_service.attributes[ + f"{SHELL_NAME}.Github Terraform Module URL"] = url + self.integration_data1.context.resource.attributes[ + f"{SHELL_NAME}.Branch"] = branch + self._driver_helper.tf_service.attributes[ + f"{SHELL_NAME}.Branch"] = branch + + downloader = Downloader(self._driver_helper) + tf_workingdir = downloader.download_terraform_module() + self.assertTrue(os.path.exists(os.path.join(tf_workingdir, TF_HELLO_FILE))) + + def test_public_and_private_hello_dl(self): + self._test_download_terraform_module(GH_TF_PUBLIC_HELLO_URL_FILE, "") + self._test_download_terraform_module(os.environ.get("GITHUB_TF_PRIVATE_HELLO_URL"), "") + self._test_download_terraform_module(GH_TF_PUBLIC_HELLO_URL_FOLDER, "") + + def test_download_terraform_executable(self): + downloader = Downloader(self._driver_helper) + tf_workingdir = downloader.download_terraform_module() + downloader.download_terraform_executable(tf_workingdir) + + self.assertTrue(os.path.exists(os.path.join(tf_workingdir, TERRAFORM_EXEC_FILE))) + self.assertTrue(os.access(os.path.join(tf_workingdir, TERRAFORM_EXEC_FILE), os.X_OK)) diff --git a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py index b36ab0e6..f86f8a9e 100644 --- a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py @@ -1,19 +1,17 @@ from unittest.mock import patch, Mock - -# from cloudshell.api.cloudshell_api import NameValuePair -from cloudshell.api.cloudshell_api import NameValuePair from dotenv import load_dotenv -from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER, INT_TEST_CLP_RESOURSE -from typing import Callable - -from tests.integration_tests.helper_objects.integration_context import MockAPIIntegrationData import os from unittest import TestCase - from pathlib import Path -from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder +from package.tests.integration_tests.helper_services.mock_tests_prepper import pre_exec_azure_vault, pre_destroy +from package.tests.integration_tests.helper_objects.mock_tests_data import MockTestsData + +from package.tests.constants import MOCK_ALIAS1 + +from package.tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, EXPECTED_VAULT_TF_OUTPUTS, \ + EXPECTED_VAULT_TF_SENSETIVE_OUTPUTS_DEC class TestMockTerraformExecuteDestroy(TestCase): @@ -26,55 +24,9 @@ def setUpClass(self): @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') def setUp(self, patched_api) -> None: - - self._prepare_mock_api() - patched_api.return_value.get_api.return_value = self.mock_api - - self._mocked_tf_working_dir = '' - self._prepare_mock_services() - - self.mock_api.GetReservationDetails.return_value.ReservationDescription.Services = [self._service1, - self._service2] - self._prepare_integration_data() - - def _prepare_integration_data(self): - self.integration_data1 = MockAPIIntegrationData(self._service1.Alias, self.mock_api) - self.integration_data2 = MockAPIIntegrationData(self._service2.Alias, self.mock_api) - - def _prepare_mock_services(self): - self._service1 = Mock() - self._service1.Alias = "MOCK_ALIAS1" - self._service1.Attributes = ServiceAttributesMockBuilder.create_empty_attributes() - self._service2 = Mock() - self._service2.Alias = "MOCK_ALIAS2" - self._service2.Attributes = ServiceAttributesMockBuilder.create_empty_attributes() - - def _prepare_mock_api(self): - self.mock_api = Mock() - self.mock_api.DecryptPassword = _decrypt_password - self.mock_api.GetResourceDetails.return_value.ResourceFamilyName = 'Cloud Provider' - self.mock_api.GetResourceDetails.return_value.ResourceModelName = 'Microsoft Azure' - self.mock_api.GetResourceDetails.return_value.ResourceAttributes = [ - NameValuePair(Name="Azure Subscription ID", Value=os.environ.get("AZURE_SUBSCRIPTION_ID")), - NameValuePair(Name="Azure Tenant ID", Value=os.environ.get("AZURE_TENANT_ID")), - NameValuePair(Name="Azure Application ID", Value=os.environ.get("AZURE_APPLICATION_ID")), - NameValuePair(Name="Azure Application Key", Value=os.environ.get("AZURE_APPLICATION_KEY_DEC")) - ] - - '''------------------------------ Generic Execute/Destroy functions ---------------------------------''' - - def run_execute(self, pre_exec_function: Callable, integration_data: MockAPIIntegrationData): - self.pre_exec_prep(pre_exec_function, integration_data) - integration_data.tf_shell.execute_terraform() - - def run_destroy(self, pre_destroy_function: Callable, integration_data: MockAPIIntegrationData): - self.pre_destroy_prep(pre_destroy_function, integration_data) - integration_data.tf_shell.destroy_terraform() - - def run_execute_and_destroy(self, pre_exec_function: Callable, pre_destroy_function: Callable, - integration_data: MockAPIIntegrationData): - self.run_execute(pre_exec_function, integration_data) - self.run_destroy(pre_destroy_function, integration_data) + self.test_data_object = MockTestsData() + patched_api.return_value.get_api.return_value = self.test_data_object.mock_api + self.test_data_object.prepare_integration_data() '''------------------------------ Test Cases ---------------------------------''' @@ -83,151 +35,32 @@ def run_execute_and_destroy(self, pre_exec_function: Callable, pre_destroy_funct @patch('cloudshell.iac.terraform.services.object_factory.CloudShellSessionContext') def test_execute_and_destroy_azure_vault(self, patch_api, patched_sbdata_handler, can_destroy_run): can_destroy_run.return_value = True - patch_api.return_value.get_api.return_value = self.mock_api + patch_api.return_value.get_api.return_value = self.test_data_object.mock_api mock_sbdata_handler = Mock() - mock_sbdata_handler.get_tf_working_dir = self._get_mocked_tf_working_dir - mock_sbdata_handler.set_tf_working_dir = self._set_mocked_tf_working_dir + mock_sbdata_handler.get_tf_working_dir = self.test_data_object._get_mocked_tf_working_dir + mock_sbdata_handler.set_tf_working_dir = self.test_data_object._set_mocked_tf_working_dir patched_sbdata_handler.return_value = mock_sbdata_handler - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - - '''------------------------------ Functions : general _pre prep functions ---------------------------------''' - - def pre_exec_prep(self, pre_exec_function: Callable, integration_data: MockAPIIntegrationData): - pre_exec_function(integration_data) - - def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: MockAPIIntegrationData): - pre_destroy_function(integration_data) - - def clear_sb_data(self): - self.integration_data1.api.ClearSandboxData(self.integration_data1.context.reservation.reservation_id) - - '''------------------------------ Functions : prep before exec -------------------------------------------''' - - def pre_exec(self, integration_data: MockAPIIntegrationData): - pass - - def pre_exec_azure_vault(self, integration_data: MockAPIIntegrationData): - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Terraform Inputs", - os.environ.get("AZUREAPP_TF_INPUTS"), - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TOKEN}", - os.environ.get("GH_TOKEN_DEC"), - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", - INT_TEST_TF_VER, - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.CLOUD_PROVIDER}", - INT_TEST_CLP_RESOURSE, - integration_data - ) - self._prepare_service1_before_execute(integration_data) - - def _prepare_service1_before_execute(self, integration_data): - service1 = Mock() - service1.Alias = integration_data.context.resource.name - service1.Attributes = integration_data.context.resource.attributes - self.mock_api.GetReservationDetails.return_value.ReservationDescription.Services = [service1] - integration_data.create_tf_shell() - - def pre_exec_azure_mssql(self, integration_data: MockAPIIntegrationData): - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Terraform Inputs", - os.environ.get("AZUREMSSQL_TF_INPUTS"), - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Github Terraform Module URL", - os.environ.get("GH_TF_PRIVATE_AZUREMSSQL_URL"), - integration_data - ) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.UUID", - os.environ.get(""), - integration_data - ) - self._prepare_service1_before_execute(integration_data) - - def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: MockAPIIntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_ACCESS_KEY"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: MockAPIIntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_CLOUD_CRED"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: MockAPIIntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: MockAPIIntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get("REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE"), - integration_data - ) - - def pre_exec_azure_vault_without_remote(self, integration_data: MockAPIIntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_mock_service( - f"{SHELL_NAME}.Remote State Provider", - os.environ.get(""), - integration_data - ) - - '''------------------------------ Functions : prep before destroy -----------------------------------------''' - - def pre_destroy(self, integration_data: MockAPIIntegrationData): - # As UUID has been created and SB data now contains UUID and Status we must update context so destroy can run - for attribute in integration_data.context.resource.attributes: - if attribute.Name == f"{SHELL_NAME}.UUID": - attribute.Value = integration_data.tf_shell._tf_service.attributes[f"{SHELL_NAME}.UUID"] - - '''------------------------------ Helper Functions ---------------------------------------------------------''' - @staticmethod - def _set_attribute_on_mock_service(attr_name: str, attr_value: str, integration_data: MockAPIIntegrationData): - for attribute in integration_data.context.resource.attributes: - if attribute.Name == attr_name: - attribute.Value = attr_value - return - - def _get_mocked_tf_working_dir(self): - return self._mocked_tf_working_dir - - def _set_mocked_tf_working_dir(self, tf_working_dir: str): - self._mocked_tf_working_dir = tf_working_dir - -def _decrypt_password(x): - # result = mock.MagicMock() - result = Mock() - result.Value = x - return result + pre_exec_azure_vault(self.test_data_object) + self.test_data_object.integration_data1.tf_shell.execute_terraform() + + self.assertTrue(self.are_vault_tf_outputs_correct()) + pre_destroy(self.test_data_object.integration_data1) + self.test_data_object.integration_data1.tf_shell.destroy_terraform() + + def are_vault_tf_outputs_correct(self): + tf_outputs_correct = False + tf_sensitive_outputs_correct = False + try: + for call in self.test_data_object.mock_api.SetServiceAttributesValues.call_args_list: + if call.args[1] == MOCK_ALIAS1: + for attribute_name_value in call.args[2]: + if attribute_name_value.Name == f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_OUTPUTS}" and \ + attribute_name_value.Value == EXPECTED_VAULT_TF_OUTPUTS: + tf_outputs_correct = True + if attribute_name_value.Name == f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_SENSIITVE_OUTPUTS}" and \ + attribute_name_value.Value == EXPECTED_VAULT_TF_SENSETIVE_OUTPUTS_DEC: + tf_sensitive_outputs_correct = True + except Exception as e: + raise + return tf_outputs_correct and tf_sensitive_outputs_correct diff --git a/package/tests/integration_tests/real_api_based_tests/int_test_real_downloader.py b/package/tests/integration_tests/real_api_based_tests/int_test_real_downloader.py index cc507d38..bbff9956 100644 --- a/package/tests/integration_tests/real_api_based_tests/int_test_real_downloader.py +++ b/package/tests/integration_tests/real_api_based_tests/int_test_real_downloader.py @@ -1,5 +1,8 @@ import os from unittest import TestCase +from pathlib import Path + +from dotenv import load_dotenv from cloudshell.iac.terraform.downloaders.downloader import Downloader from cloudshell.iac.terraform.services.live_status_updater import LiveStatusUpdater @@ -8,40 +11,48 @@ from cloudshell.iac.terraform.models.shell_helper import ShellHelperObject from package.tests.constants import GH_TF_PUBLIC_HELLO_URL_FILE, GH_TF_PUBLIC_HELLO_URL_FOLDER, TERRAFORM_EXEC_FILE, \ SHELL_NAME, TF_HELLO_FILE -from package.tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData +# from package.tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData +from package.tests.integration_tests.helper_objects.integration_context import RealAPIIntegrationData from cloudshell.iac.terraform.tagging.tags import TagsManager from cloudshell.iac.terraform.services.svc_attribute_handler import ServiceAttrHandler class TestTerraformDownloader(TestCase): + @classmethod + def setUpClass(self): + load_dotenv(Path('../int_tests.env')) + if os.path.isfile(Path('../int_tests_secrets.env')): + load_dotenv(Path('../int_tests_secrets.env')) + def setUp(self) -> None: - self.integration_data = IntegrationData() + self.integration_data1 = RealAPIIntegrationData(os.environ.get("SB_SERVICE_ALIAS1")) + self.integration_data2 = RealAPIIntegrationData(os.environ.get("SB_SERVICE_ALIAS2")) - service_resource = GenericTerraformService.create_from_context(self.integration_data.context) + service_resource = GenericTerraformService.create_from_context(self.integration_data1.context) sandbox_messages = SandboxMessagesService( - self.integration_data.real_api, - self.integration_data.context.reservation.reservation_id, - self.integration_data.context.resource.name, + self.integration_data1.api, + self.integration_data1.context.reservation.reservation_id, + self.integration_data1.context.resource.name, False ) live_status_updater = LiveStatusUpdater( - self.integration_data.real_api, - self.integration_data.context.reservation.reservation_id, + self.integration_data1.api, + self.integration_data1.context.reservation.reservation_id, False ) - default_tags = TagsManager(self.integration_data.context.reservation) + default_tags = TagsManager(self.integration_data1.context.reservation) attr_handler = ServiceAttrHandler(service_resource) self._driver_helper = ShellHelperObject( - self.integration_data.real_api, - self.integration_data.context.reservation.reservation_id, + self.integration_data1.api, + self.integration_data1.context.reservation.reservation_id, service_resource, - self.integration_data._logger, + self.integration_data1._logger, sandbox_messages, live_status_updater, attr_handler, @@ -49,11 +60,11 @@ def setUp(self) -> None: ) def _test_download_terraform_module(self, url: str, branch: str): - self.integration_data.context.resource.attributes[ + self.integration_data1.context.resource.attributes[ f"{SHELL_NAME}.Github Terraform Module URL"] = url self._driver_helper.tf_service.attributes[ f"{SHELL_NAME}.Github Terraform Module URL"] = url - self.integration_data.context.resource.attributes[ + self.integration_data1.context.resource.attributes[ f"{SHELL_NAME}.Branch"] = branch self._driver_helper.tf_service.attributes[ f"{SHELL_NAME}.Branch"] = branch diff --git a/package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py b/package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py index d145bbe5..066d7777 100644 --- a/package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py +++ b/package/tests/integration_tests/real_api_based_tests/int_test_real_tf_execute_destroy.py @@ -1,14 +1,19 @@ -from cloudshell.api.cloudshell_api import AttributeNameValue from dotenv import load_dotenv -from typing import Callable -from tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, INT_TEST_TF_VER -from tests.integration_tests.helper_objects.integration_context import RealAPIIntegrationData +from package.tests.integration_tests.constants import SHELL_NAME, ATTRIBUTE_NAMES, EXPECTED_VAULT_TF_OUTPUTS, \ + EXPECTED_VAULT_TF_SENSETIVE_OUTPUTS_ENC +from package.tests.integration_tests.helper_objects.real_tests_data import RealTestsData +from package.tests.integration_tests.helper_services.real_tests_prepper import pre_exec_azure_vault, \ + pre_exec_azure_vault_with_remote_access_key_based, pre_exec_azure_vault_with_remote_cloud_cred_based, \ + pre_exec_azure_vault_with_remote_invalid_nonexistent, pre_exec_azure_vault_with_remote_invalid_wrong, \ + post_vault_cleanup import os from pathlib import Path from unittest import TestCase +from package.tests.integration_tests.helper_objects.integration_context import RealAPIIntegrationData + class TestRealTerraformExecuteDestroy(TestCase): @classmethod @@ -18,204 +23,89 @@ def setUpClass(self): load_dotenv(Path('../int_tests_secrets.env')) def setUp(self) -> None: - self.integration_data1 = RealAPIIntegrationData(os.environ.get("SB_SERVICE_ALIAS1")) - self.integration_data2 = RealAPIIntegrationData(os.environ.get("SB_SERVICE_ALIAS2")) - - def run_execute_and_destroy( - self, pre_exec_function: Callable, - pre_destroy_function: Callable, - integration_data: RealAPIIntegrationData - ): - self.clear_sb_data() - self.run_execute(pre_exec_function, integration_data) - self.run_destroy(pre_destroy_function, integration_data) - - def run_execute(self, pre_exec_function: Callable, integration_data: RealAPIIntegrationData): - self.pre_exec_prep(pre_exec_function, integration_data) - integration_data.tf_shell.execute_terraform() - - def run_destroy(self, pre_destroy_function: Callable, integration_data: RealAPIIntegrationData): - self.pre_destroy_prep(pre_destroy_function, integration_data) - integration_data.tf_shell.destroy_terraform() + self.test_data_object = RealTestsData() '''------------------------------ Test Cases ---------------------------------''' + # To be used with any preconfigured attributes set on the service in real CS instance def test_execute_and_destroy(self): - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - - def test_execute_and_destroy_azure_vault(self): - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - + self.test_data_object.clear_sb_data() + self.test_data_object.integration_data1.tf_shell.execute_terraform() + self.test_data_object.integration_data1.set_context_resource_attributes_from_cs( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}") + self.test_data_object.integration_data1.tf_shell.destroy_terraform() + + def est_execute_and_destroy_azure_vault(self): + self.test_data_object.clear_sb_data() + pre_exec_azure_vault(self.test_data_object.integration_data1) + self.test_data_object.integration_data1.tf_shell.execute_terraform() + self.test_data_object.integration_data1.set_context_resource_attributes_from_cs( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}") + self.assertTrue(self.are_vault_tf_outputs_correct(self.test_data_object.integration_data1)) + self.test_data_object.integration_data1.tf_shell.destroy_terraform() + + ''' def test_execute_dual_mssql(self): self.clear_sb_data() try: - self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data1) + self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.test_data_object.integration_data1) except Exception as e: pass try: - self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data2) + self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.test_data_object.integration_data2) except Exception as e: print("Error") pass - self.run_destroy(pre_destroy_function=self.pre_destroy, integration_data=self.integration_data1) - self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.integration_data2) + self.run_destroy(pre_destroy_function=self.pre_destroy, integration_data=self.test_data_object.integration_data1) + self.run_execute(pre_exec_function=self.pre_exec_azure_mssql, integration_data=self.test_data_object.integration_data2) + ''' def test_execute_and_destroy_azure_vault_with_remote_access_key_based(self): - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault_with_remote_access_key_based, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) + self.test_data_object.clear_sb_data() + pre_exec_azure_vault_with_remote_access_key_based(self.test_data_object.integration_data1) + self.test_data_object.integration_data1.tf_shell.execute_terraform() + self.test_data_object.integration_data1.set_context_resource_attributes_from_cs( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}") + self.assertTrue(self.are_vault_tf_outputs_correct(self.test_data_object.integration_data1)) + self.test_data_object.integration_data1.tf_shell.destroy_terraform() + post_vault_cleanup(self.test_data_object.integration_data1) def test_execute_and_destroy_azure_vault_with_remote_access_cloud_cred_based(self): - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault_with_remote_cloud_cred_based, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - - def test_execute_and_destroy_azure_vault_with_remote_invalid_nonexistent(self): - try: - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault_with_remote_invalid_nonexistent, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - except Exception as e: - pass + self.test_data_object.clear_sb_data() + pre_exec_azure_vault_with_remote_cloud_cred_based(self.test_data_object.integration_data1) + self.test_data_object.integration_data1.tf_shell.execute_terraform() + self.test_data_object.integration_data1.set_context_resource_attributes_from_cs( + f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}") + self.assertTrue(self.are_vault_tf_outputs_correct(self.test_data_object.integration_data1)) + self.test_data_object.integration_data1.tf_shell.destroy_terraform() + post_vault_cleanup(self.test_data_object.integration_data1) + + def test_execute_azure_vault_with_remote_invalid_nonexistent(self): + self.test_data_object.clear_sb_data() + pre_exec_azure_vault_with_remote_invalid_nonexistent(self.test_data_object.integration_data1) + with self.assertRaises(ValueError): + self.test_data_object.integration_data1.tf_shell.execute_terraform() + post_vault_cleanup(self.test_data_object.integration_data1) + print("") def test_execute_and_destroy_azure_vault_with_remote_invalid_wrong(self): + self.test_data_object.clear_sb_data() + pre_exec_azure_vault_with_remote_invalid_wrong(self.test_data_object.integration_data1) + with self.assertRaises(ValueError): + self.test_data_object.integration_data1.tf_shell.execute_terraform() + post_vault_cleanup(self.test_data_object.integration_data1) + print("") + + @staticmethod + def are_vault_tf_outputs_correct(integration_data: RealAPIIntegrationData): + tf_outputs_correct = False + tf_sensitive_outputs_correct = False + attrs = integration_data.tf_shell.get_tf_service().attributes try: - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault_with_remote_invalid_wrong, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) + if attrs[f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_OUTPUTS}"] == EXPECTED_VAULT_TF_OUTPUTS: + tf_outputs_correct = True + if attrs[f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_SENSIITVE_OUTPUTS}"] == EXPECTED_VAULT_TF_SENSETIVE_OUTPUTS_ENC: + tf_sensitive_outputs_correct = True except Exception as e: - pass - - def test_execute_and_destroy_azure_vault_without_remote(self): - self.run_execute_and_destroy( - pre_exec_function=self.pre_exec_azure_vault_without_remote, - pre_destroy_function=self.pre_destroy, - integration_data=self.integration_data1 - ) - - '''------------------------------ Functions : general _pre prep functions ---------------------------------''' - - def pre_exec_prep(self, pre_exec_function: Callable, integration_data: RealAPIIntegrationData): - pre_exec_function(integration_data) - - def pre_destroy_prep(self, pre_destroy_function: Callable, integration_data: RealAPIIntegrationData): - pre_destroy_function(integration_data) - - def clear_sb_data(self): - self.integration_data1.api.ClearSandboxData(self.integration_data1.context.reservation.reservation_id) - - '''------------------------------ Functions : prep before exec -------------------------------------------''' - - def pre_exec(self, integration_data: RealAPIIntegrationData): - pass - - def pre_exec_azure_vault(self, integration_data: RealAPIIntegrationData): - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", - os.environ.get("AZUREAPP_TF_INPUTS"), - integration_data - ) - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", - os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), - integration_data - ) - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TERRAFORM_VERSION}", - INT_TEST_TF_VER, - integration_data - ) - self._set_attribute_on_service( - f"{SHELL_NAME}.UUID", - os.environ.get(""), - integration_data - ) - - def pre_exec_azure_mssql(self, integration_data: RealAPIIntegrationData): - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", - os.environ.get("AZUREMSSQL_TF_INPUTS"), - integration_data - ) - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", - os.environ.get("GITHUB_TF_PRIVATE_AZUREMSSQL_URL"), - integration_data - ) - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}", - os.environ.get(""), - integration_data - ) - - def pre_exec_azure_vault_with_remote_access_key_based(self, integration_data: RealAPIIntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", - os.environ.get("REMOTE_STATE_PROVIDER_ACCESS_KEY"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_cloud_cred_based(self, integration_data: RealAPIIntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", - os.environ.get("REMOTE_STATE_PROVIDER_CLOUD_CRED"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_invalid_nonexistent(self, integration_data: RealAPIIntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", - os.environ.get("REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE"), - integration_data - ) - - def pre_exec_azure_vault_with_remote_invalid_wrong(self, integration_data: RealAPIIntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", - os.environ.get("REMOTE_STATE_PROVIDER_INVALID_WRONG_RESOURCE"), - integration_data - ) - - def pre_exec_azure_vault_without_remote(self, integration_data: RealAPIIntegrationData): - self.pre_exec_azure_vault(integration_data) - self._set_attribute_on_service( - f"{SHELL_NAME}.{ATTRIBUTE_NAMES.REMOTE_STATE_PROVIDER}", - os.environ.get(""), - integration_data - ) - - '''------------------------------ Functions : prep before destroy -----------------------------------------''' - - def pre_destroy(self, integration_data: RealAPIIntegrationData): - # As UUID has been created and SB data now contains UUID and Status we must update context so destroy can run - integration_data.set_context_resource_attributes_from_cs(f"{SHELL_NAME}.{ATTRIBUTE_NAMES.UUID}") - - '''------------------------------ Helper Functions ---------------------------------------------------------''' - - def _set_attribute_on_service(self, attr_name: str, attr_value: str, integration_data: RealAPIIntegrationData): - attr_req = [AttributeNameValue(attr_name, attr_value)] - integration_data.api.SetServiceAttributesValues( - integration_data.context.reservation.reservation_id, - integration_data.context.resource.name, - attr_req - ) + raise + return tf_outputs_correct and tf_sensitive_outputs_correct diff --git a/package/tests/integration_tests/tf_modules/azure-paas-mssql/main.tf b/package/tests/tests_helper_files/tf_modules/azure-paas-mssql/main.tf similarity index 100% rename from package/tests/integration_tests/tf_modules/azure-paas-mssql/main.tf rename to package/tests/tests_helper_files/tf_modules/azure-paas-mssql/main.tf diff --git a/package/tests/integration_tests/tf_modules/azure-paas-mssql/runSQLCMD.sh b/package/tests/tests_helper_files/tf_modules/azure-paas-mssql/runSQLCMD.sh similarity index 100% rename from package/tests/integration_tests/tf_modules/azure-paas-mssql/runSQLCMD.sh rename to package/tests/tests_helper_files/tf_modules/azure-paas-mssql/runSQLCMD.sh diff --git a/package/tests/integration_tests/tf_modules/azure-paas-mssql/vars.tf b/package/tests/tests_helper_files/tf_modules/azure-paas-mssql/vars.tf similarity index 100% rename from package/tests/integration_tests/tf_modules/azure-paas-mssql/vars.tf rename to package/tests/tests_helper_files/tf_modules/azure-paas-mssql/vars.tf diff --git a/package/tests/integration_tests/tf_modules/azure-vault/main.tf b/package/tests/tests_helper_files/tf_modules/azure-vault/main.tf similarity index 100% rename from package/tests/integration_tests/tf_modules/azure-vault/main.tf rename to package/tests/tests_helper_files/tf_modules/azure-vault/main.tf diff --git a/package/tests/integration_tests/tf_modules/azure-vault/vars.tf b/package/tests/tests_helper_files/tf_modules/azure-vault/vars.tf similarity index 100% rename from package/tests/integration_tests/tf_modules/azure-vault/vars.tf rename to package/tests/tests_helper_files/tf_modules/azure-vault/vars.tf From eecda88c3fe838741657357c5a14526b56651527 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 13:34:04 -0600 Subject: [PATCH 57/70] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c94bde10..fb13e175 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} - CURRENT_BRANCH = ${GITHUB_REF##*/} + #CURRENT_BRANCH = ${GITHUB_REF##*/} run: | pwd From d10288771b014d695c5a1227431b19472cae674f Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 13:40:28 -0600 Subject: [PATCH 58/70] temp remove azure-validator --- package/test_requirements.txt | 4 ++-- .../integration_tests/helper_services/azure_validator.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/package/test_requirements.txt b/package/test_requirements.txt index e81be25f..6d1b1bd7 100644 --- a/package/test_requirements.txt +++ b/package/test_requirements.txt @@ -2,5 +2,5 @@ mock==4.0.3 python-dotenv==0.18.0 pathlib==1.0.1 -azure-common==1.1.27 -azure-mgmt==5.0.0 \ No newline at end of file +#azure-common==1.1.27 +#azure-mgmt==5.0.0 \ No newline at end of file diff --git a/package/tests/integration_tests/helper_services/azure_validator.py b/package/tests/integration_tests/helper_services/azure_validator.py index cf28e7c8..269caee5 100644 --- a/package/tests/integration_tests/helper_services/azure_validator.py +++ b/package/tests/integration_tests/helper_services/azure_validator.py @@ -1,3 +1,4 @@ +''' import os from azure.common.credentials import ServicePrincipalCredentials from azure.mgmt.resource import ResourceManagementClient @@ -17,4 +18,5 @@ def __init__(self): self._client = ResourceManagementClient(self._credentials, self._subscription_id) for item in self._client.resource_groups.list(): - print(item) \ No newline at end of file + print(item) +''' From 99572d196d01a3333e74a56764de0323f118e15d Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 13:52:02 -0600 Subject: [PATCH 59/70] lint fixes --- package/tests/constants.py | 2 -- package/tests/integration_tests/constants.py | 3 ++- .../helper_objects/integration_context.py | 3 --- .../integration_tests/helper_objects/mock_tests_data.py | 8 -------- .../integration_tests/helper_objects/real_tests_data.py | 8 ++------ .../mock_api_based_tests/int_test_mock_downloader.py | 2 -- .../int_test_mock_tf_execute_destroy.py | 1 - 7 files changed, 4 insertions(+), 23 deletions(-) diff --git a/package/tests/constants.py b/package/tests/constants.py index b65b82cd..f41e9e21 100644 --- a/package/tests/constants.py +++ b/package/tests/constants.py @@ -11,5 +11,3 @@ MOCK_ALIAS1 = "MOCK_ALIAS1" MOCK_ALIAS2 = "MOCK_ALIAS2" - - diff --git a/package/tests/integration_tests/constants.py b/package/tests/integration_tests/constants.py index 37b0a083..c7fd8d74 100644 --- a/package/tests/integration_tests/constants.py +++ b/package/tests/integration_tests/constants.py @@ -20,6 +20,7 @@ class ATTRIBUTE_NAMES: CLOUD_PROVIDER = "Cloud Provider" UUID = "UUID" + EXPECTED_VAULT_TF_OUTPUTS = "BLA1=bla1,BLA2=bla2" EXPECTED_VAULT_TF_SENSETIVE_OUTPUTS_DEC = "SECRET_VALUE=test,SECRET_VALUE_2=my_secret" -EXPECTED_VAULT_TF_SENSETIVE_OUTPUTS_ENC = 'nwH0Qhxnn7ntDpWxsvol+89EPFFSvYE16gHjACzsQRDzvXXnNFc4WfYpgq5w1kw8' \ No newline at end of file +EXPECTED_VAULT_TF_SENSETIVE_OUTPUTS_ENC = 'nwH0Qhxnn7ntDpWxsvol+89EPFFSvYE16gHjACzsQRDzvXXnNFc4WfYpgq5w1kw8' diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index b44d8e11..ef561f8b 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -6,15 +6,12 @@ from cloudshell.logging.qs_logger import get_qs_logger from cloudshell.shell.core.driver_context import ResourceCommandContext -# from tests.integration_tests.helper_objects.env_vars import EnvVars from package.tests.integration_tests.helper_objects.env_vars import EnvVars -# from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder from abc import ABCMeta - class BaseIntegrationData(metaclass=ABCMeta): def __init__(self, service_name: str): self._sb_service_alias = service_name diff --git a/package/tests/integration_tests/helper_objects/mock_tests_data.py b/package/tests/integration_tests/helper_objects/mock_tests_data.py index e4ef9851..ac2e4197 100644 --- a/package/tests/integration_tests/helper_objects/mock_tests_data.py +++ b/package/tests/integration_tests/helper_objects/mock_tests_data.py @@ -56,16 +56,8 @@ def _get_mocked_tf_working_dir(self): def _set_mocked_tf_working_dir(self, tf_working_dir: str): self._mocked_tf_working_dir = tf_working_dir - ''' - @staticmethod - def _set_service_attributes_values(sandbox_id: str, tf_service_name: str, - attr_update_req: list[AttributeNameValue]): - print("") - pass - ''' def _decrypt_password(x): result = Mock() result.Value = x return result - diff --git a/package/tests/integration_tests/helper_objects/real_tests_data.py b/package/tests/integration_tests/helper_objects/real_tests_data.py index 685e8e04..b5c678e5 100644 --- a/package/tests/integration_tests/helper_objects/real_tests_data.py +++ b/package/tests/integration_tests/helper_objects/real_tests_data.py @@ -1,11 +1,7 @@ import os -from unittest.mock import Mock -from package.tests.integration_tests.helper_objects.integration_context import MockAPIIntegrationData, \ - RealAPIIntegrationData -from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder +from package.tests.integration_tests.helper_objects.integration_context import RealAPIIntegrationData -from cloudshell.api.cloudshell_api import NameValuePair class RealTestsData(object): def __init__(self): @@ -13,4 +9,4 @@ def __init__(self): self.integration_data2 = RealAPIIntegrationData(os.environ.get("SB_SERVICE_ALIAS2")) def clear_sb_data(self): - self.integration_data1.api.ClearSandboxData(self.integration_data1.context.reservation.reservation_id) \ No newline at end of file + self.integration_data1.api.ClearSandboxData(self.integration_data1.context.reservation.reservation_id) diff --git a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py index 25d9f649..b62c92ce 100644 --- a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py @@ -38,8 +38,6 @@ def setUp(self, patched_api) -> None: self._service2] self._prepare_integration_data() - - service_resource = GenericTerraformService.create_from_context(self.integration_data1.context) sandbox_messages = SandboxMessagesService( diff --git a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py index 70acc0ee..6a80a066 100644 --- a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py @@ -47,7 +47,6 @@ def test_execute_and_destroy_azure_vault(self, patch_api, patched_sbdata_handler mock_sbdata_handler.set_tf_working_dir = self.test_data_object._set_mocked_tf_working_dir patched_sbdata_handler.return_value = mock_sbdata_handler - pre_exec_azure_vault(self.test_data_object) self.test_data_object.integration_data1.tf_shell.execute_terraform() From ccb235d1afa2a24a29051e3956334651107c3b61 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 13:53:36 -0600 Subject: [PATCH 60/70] forgot to import abstractmethod --- .../integration_tests/helper_objects/integration_context.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package/tests/integration_tests/helper_objects/integration_context.py b/package/tests/integration_tests/helper_objects/integration_context.py index ef561f8b..01513859 100644 --- a/package/tests/integration_tests/helper_objects/integration_context.py +++ b/package/tests/integration_tests/helper_objects/integration_context.py @@ -6,10 +6,12 @@ from cloudshell.logging.qs_logger import get_qs_logger from cloudshell.shell.core.driver_context import ResourceCommandContext +# from tests.integration_tests.helper_objects.env_vars import EnvVars from package.tests.integration_tests.helper_objects.env_vars import EnvVars +# from tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder -from abc import ABCMeta +from abc import ABCMeta, abstractmethod class BaseIntegrationData(metaclass=ABCMeta): From 2597d195a070c96e9b688a147b2a81ec0d63fcc2 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 14:08:05 -0600 Subject: [PATCH 61/70] lint --- .../tests/integration_tests/helper_objects/mock_tests_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package/tests/integration_tests/helper_objects/mock_tests_data.py b/package/tests/integration_tests/helper_objects/mock_tests_data.py index ac2e4197..48a63123 100644 --- a/package/tests/integration_tests/helper_objects/mock_tests_data.py +++ b/package/tests/integration_tests/helper_objects/mock_tests_data.py @@ -4,7 +4,7 @@ from package.tests.integration_tests.helper_objects.integration_context import MockAPIIntegrationData from package.tests.integration_tests.helper_services.service_attributes_factory import ServiceAttributesMockBuilder -from cloudshell.api.cloudshell_api import NameValuePair, AttributeNameValue +from cloudshell.api.cloudshell_api import NameValuePair class MockTestsData(object): From 41803431a1cca8876873f0feda061e4b6328de82 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 14:17:00 -0600 Subject: [PATCH 62/70] Update ci.yml --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb13e175..b7ac1e0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,4 @@ jobs: run: | pwd - cd package/tests/integration_tests/mock_api_based_tests - pwd - python -m unittest discover -p "int_test_mock*.py" + python -m unittest discover -p "package/tests/integration_tests/mock_api_based_tests/int_test_mock*.py" From 7f7d0fcfd954a1aebb1c0e24861d526bad9b8b22 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 14:35:07 -0600 Subject: [PATCH 63/70] testing current branch --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7ac1e0f..4c3f05dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,8 +54,9 @@ jobs: AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} - #CURRENT_BRANCH = ${GITHUB_REF##*/} + CURRENT_BRANCH = ${{ env.GITHUB_HEAD_REF_SLUG }} run: | pwd + echo " - ${{ env.GITHUB_HEAD_REF_SLUG }}" python -m unittest discover -p "package/tests/integration_tests/mock_api_based_tests/int_test_mock*.py" From 282ed02bf6df7bc6935cbdbf2bcd92cab5f3de41 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 14:41:49 -0600 Subject: [PATCH 64/70] testing current branch in CI --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c3f05dc..368d1222 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,13 +50,14 @@ jobs: - name: Test integration tests for package/tests/integration_tests env: GH_TOKEN_DEC: ${{ secrets.GH_TOKEN_DEC }} + CURRENT_BRANCH: ${{ env.GITHUB_HEAD_REF_SLUG }} AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_APPLICATION_ID: ${{ secrets.AZURE_APPLICATION_ID }} - CURRENT_BRANCH = ${{ env.GITHUB_HEAD_REF_SLUG }} + run: | pwd - echo " - ${{ env.GITHUB_HEAD_REF_SLUG }}" + echo "The branch is: ${{ env.GITHUB_HEAD_REF_SLUG }}" python -m unittest discover -p "package/tests/integration_tests/mock_api_based_tests/int_test_mock*.py" From 4937142333c18def6c66a7319523b76cf18e4be5 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 14:48:10 -0600 Subject: [PATCH 65/70] Update ci.yml --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 368d1222..d4753592 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,5 +59,6 @@ jobs: run: | pwd - echo "The branch is: ${{ env.GITHUB_HEAD_REF_SLUG }}" + echo "1: The branch is: ${{ env.GITHUB_HEAD_REF_SLUG }}" + echo "2: The branch is: ${GITHUB_REF##*/} python -m unittest discover -p "package/tests/integration_tests/mock_api_based_tests/int_test_mock*.py" From 3cd57398f436468f6f9defd434b40b14c1551165 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 14:51:21 -0600 Subject: [PATCH 66/70] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4753592..6437a201 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,5 +60,5 @@ jobs: run: | pwd echo "1: The branch is: ${{ env.GITHUB_HEAD_REF_SLUG }}" - echo "2: The branch is: ${GITHUB_REF##*/} + echo "2: The branch is: ${{ github.head_ref }} python -m unittest discover -p "package/tests/integration_tests/mock_api_based_tests/int_test_mock*.py" From 6f19fff4a264366c211b37e8403ad7b5e0ec278c Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 14:52:54 -0600 Subject: [PATCH 67/70] Update ci.yml --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6437a201..4d083228 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,5 @@ jobs: run: | pwd - echo "1: The branch is: ${{ env.GITHUB_HEAD_REF_SLUG }}" - echo "2: The branch is: ${{ github.head_ref }} + echo "2: The branch is: ${{ github.head_ref }}" python -m unittest discover -p "package/tests/integration_tests/mock_api_based_tests/int_test_mock*.py" From 02efc0439dd11ac47e3e0290432755bc95fb1f92 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 14:56:41 -0600 Subject: [PATCH 68/70] Update ci.yml --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d083228..80a0ca05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: - name: Test integration tests for package/tests/integration_tests env: GH_TOKEN_DEC: ${{ secrets.GH_TOKEN_DEC }} - CURRENT_BRANCH: ${{ env.GITHUB_HEAD_REF_SLUG }} + CURRENT_BRANCH: ${{ env.BRANCH_NAME }} AZURE_APPLICATION_KEY_DEC: ${{ secrets.AZURE_APPLICATION_KEY_DEC }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} @@ -59,5 +59,6 @@ jobs: run: | pwd - echo "2: The branch is: ${{ github.head_ref }}" + echo "1: ${{ env.BRANCH_NAME }}" + python -m unittest discover -p "package/tests/integration_tests/mock_api_based_tests/int_test_mock*.py" From bd5357c040466de46d77f7d394e9b204105c3ec7 Mon Sep 17 00:00:00 2001 From: qualidan Date: Wed, 29 Dec 2021 15:24:29 -0600 Subject: [PATCH 69/70] added public repo via current branch --- .../helper_services/mock_tests_prepper.py | 5 +++-- package/tests/integration_tests/int_tests.env | 4 ++++ .../integration_tests/int_tests_secrets.template.env | 4 +++- .../mock_api_based_tests/int_test_mock_downloader.py | 2 +- .../int_test_mock_tf_execute_destroy.py | 12 ++++++++---- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/package/tests/integration_tests/helper_services/mock_tests_prepper.py b/package/tests/integration_tests/helper_services/mock_tests_prepper.py index 9ae35ce2..52191cbb 100644 --- a/package/tests/integration_tests/helper_services/mock_tests_prepper.py +++ b/package/tests/integration_tests/helper_services/mock_tests_prepper.py @@ -7,7 +7,8 @@ from package.tests.integration_tests.helper_objects.mock_tests_data import MockTestsData -def pre_exec_azure_vault(mock_tests_data: MockTestsData): + +def pre_exec_azure_vault(mock_tests_data: MockTestsData, azure_vault_url: str): set_attribute_on_mock_service( f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", os.environ.get("AZUREAPP_TF_INPUTS"), @@ -15,7 +16,7 @@ def pre_exec_azure_vault(mock_tests_data: MockTestsData): ) set_attribute_on_mock_service( f"{SHELL_NAME}.{ATTRIBUTE_NAMES.GITHUB_TERRAFORM_MODULE_URL}", - os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL"), + azure_vault_url, mock_tests_data.integration_data1 ) set_attribute_on_mock_service( diff --git a/package/tests/integration_tests/int_tests.env b/package/tests/integration_tests/int_tests.env index cee0d145..ab085a06 100644 --- a/package/tests/integration_tests/int_tests.env +++ b/package/tests/integration_tests/int_tests.env @@ -3,6 +3,10 @@ AZUREAPP_TF_INPUTS = "KEYVAULT_NAME=tf-tests-keyvault,KEYVAULT_RG=cs-tf-tests-rg,SECRET_NAME=test" GITHUB_TF_PRIVATE_AZUREAPP_URL = "https://github.com/alexazarh/Colony-experiments/tree/master/terraform/azure-vault" +GITHUB_TF_PUBLIC_AZUREAPP_URL_PRE = "https://github.com/QualiSystemsLab/CloudShell-Terraform-Shell/tree/" +GITHUB_TF_PUBLIC_AZUREAPP_URL_POST = "/package/tests/tests_helper_files/tf_modules/azure-vault" + + REMOTE_STATE_PROVIDER_ACCESS_KEY = "azure_tf_backend1" REMOTE_STATE_PROVIDER_CLOUD_CRED = "azure_tf_backend2" REMOTE_STATE_PROVIDER_INVALID_NO_RESOURCE = "azure_tf_invalid" diff --git a/package/tests/integration_tests/int_tests_secrets.template.env b/package/tests/integration_tests/int_tests_secrets.template.env index 100be69c..c8e3533b 100644 --- a/package/tests/integration_tests/int_tests_secrets.template.env +++ b/package/tests/integration_tests/int_tests_secrets.template.env @@ -8,4 +8,6 @@ GH_TOKEN_DEC = "" AZURE_SUBSCRIPTION_ID = "" AZURE_TENANT_ID = "" -AZURE_APPLICATION_ID = "" \ No newline at end of file +AZURE_APPLICATION_ID = "" + +CURRENT_BRANCH = "" \ No newline at end of file diff --git a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py index b62c92ce..f4427974 100644 --- a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_downloader.py @@ -13,7 +13,7 @@ from cloudshell.iac.terraform.models.shell_helper import ShellHelperObject from package.tests.constants import GH_TF_PUBLIC_HELLO_URL_FILE, GH_TF_PUBLIC_HELLO_URL_FOLDER, TERRAFORM_EXEC_FILE, \ SHELL_NAME, TF_HELLO_FILE, MOCK_ALIAS2, MOCK_ALIAS1 -# from package.tests.integration_tests.helper_objects.downloader_integration_context import IntegrationData + from package.tests.integration_tests.helper_objects.integration_context import MockAPIIntegrationData from cloudshell.iac.terraform.tagging.tags import TagsManager diff --git a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py index 6a80a066..d29f0991 100644 --- a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py @@ -47,10 +47,14 @@ def test_execute_and_destroy_azure_vault(self, patch_api, patched_sbdata_handler mock_sbdata_handler.set_tf_working_dir = self.test_data_object._set_mocked_tf_working_dir patched_sbdata_handler.return_value = mock_sbdata_handler - pre_exec_azure_vault(self.test_data_object) - self.test_data_object.integration_data1.tf_shell.execute_terraform() - - self.assertTrue(self.are_vault_tf_outputs_correct()) + azure_vault_private_url = os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL") + azure_vault_public_url = f'{os.environ.get("GITHUB_TF_PUBLIC_AZUREAPP_URL_PRE")}' \ + f'{os.environ.get("CURRENT_BRANCH")}' \ + f'{os.environ.get("GITHUB_TF_PUBLIC_AZUREAPP_URL_POST")}' + for url in [azure_vault_private_url, azure_vault_public_url]: + pre_exec_azure_vault(self.test_data_object, url) + self.test_data_object.integration_data1.tf_shell.execute_terraform() + self.assertTrue(self.are_vault_tf_outputs_correct()) pre_destroy(self.test_data_object.integration_data1) self.test_data_object.integration_data1.tf_shell.destroy_terraform() From 3b912a7b5c9cfedc6d938a8c125c057fcf1979c5 Mon Sep 17 00:00:00 2001 From: alexazarh Date: Mon, 7 Feb 2022 11:27:38 -0600 Subject: [PATCH 70/70] fix lint --- .../integration_tests/helper_services/mock_tests_prepper.py | 1 - .../mock_api_based_tests/int_test_mock_tf_execute_destroy.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/package/tests/integration_tests/helper_services/mock_tests_prepper.py b/package/tests/integration_tests/helper_services/mock_tests_prepper.py index 52191cbb..2cdbebc5 100644 --- a/package/tests/integration_tests/helper_services/mock_tests_prepper.py +++ b/package/tests/integration_tests/helper_services/mock_tests_prepper.py @@ -7,7 +7,6 @@ from package.tests.integration_tests.helper_objects.mock_tests_data import MockTestsData - def pre_exec_azure_vault(mock_tests_data: MockTestsData, azure_vault_url: str): set_attribute_on_mock_service( f"{SHELL_NAME}.{ATTRIBUTE_NAMES.TF_INPUTS}", diff --git a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py index d29f0991..632d9c52 100644 --- a/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py +++ b/package/tests/integration_tests/mock_api_based_tests/int_test_mock_tf_execute_destroy.py @@ -49,8 +49,8 @@ def test_execute_and_destroy_azure_vault(self, patch_api, patched_sbdata_handler azure_vault_private_url = os.environ.get("GITHUB_TF_PRIVATE_AZUREAPP_URL") azure_vault_public_url = f'{os.environ.get("GITHUB_TF_PUBLIC_AZUREAPP_URL_PRE")}' \ - f'{os.environ.get("CURRENT_BRANCH")}' \ - f'{os.environ.get("GITHUB_TF_PUBLIC_AZUREAPP_URL_POST")}' + f'{os.environ.get("CURRENT_BRANCH")}' \ + f'{os.environ.get("GITHUB_TF_PUBLIC_AZUREAPP_URL_POST")}' for url in [azure_vault_private_url, azure_vault_public_url]: pre_exec_azure_vault(self.test_data_object, url) self.test_data_object.integration_data1.tf_shell.execute_terraform()