diff --git a/.github/workflows/test-deployer.yml b/.github/workflows/test-deployer.yml new file mode 100644 index 00000000..b6a80f4a --- /dev/null +++ b/.github/workflows/test-deployer.yml @@ -0,0 +1,234 @@ +# Runs the Itential Deployer on EC2 instances conforming to some validated design and other parameters +# (e.g. OS type and version), and validates that the deployed services are all running correctly. +# This allows for testing deployments on various configurations in parallel in a fully automated manner. + +# Main steps performed by this workflow: +# - Cloning Themis from GitLab +# - Provisioning EC2 instances using Themis +# - Converting OpenTofu output to Ansible inventory +# - Running deployer on newly created instances +# - Running validation script to test each service (Redis, MongoDB, Platform, Gateway) +# - Terminating all instances using Themis, irrespective of success + + +name: Test Deployer + + +on: + workflow_call: + inputs: + ref: + required: false + type: string + design: + required: true + type: string + os-type: + required: true + type: string + os-version: + required: true + type: string + secrets: + AWS_ACCESS_KEY_ID: + required: true + AWS_SECRET_ACCESS_KEY: + required: true + AWS_SESSION_TOKEN: + required: true + GITLAB_SSH_KEY: + required: true + EC2_SSH_KEY: + required: true + NEXUS_USERNAME: + required: true + NEXUS_PASSWORD: + required: true + + +jobs: + test-deployer: + runs-on: self-hosted + container: + image: ghcr.io/catthehacker/ubuntu:act-latest + env: + GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no + PIP_BREAK_SYSTEM_PACKAGES: "1" + timeout-minutes: 60 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Update package lists + run: sudo apt update + + - name: Install Python + run: | + sudo apt install -y python3 + wget https://bootstrap.pypa.io/get-pip.py && sudo python3 get-pip.py && rm get-pip.py + sudo ln -s /usr/bin/python3 /usr/bin/python || true + sudo ln -s /usr/bin/pip3 /usr/bin/pip || true + python --version + pip --version + + # For some reason, Node appears to be a dependency of Themis + - name: Install Node.js + run: | + sudo apt install -y nodejs + node --version + + - name: Install OpenTofu + uses: opentofu/setup-opentofu@v1 + with: + tofu_version: latest + + - name: Install Ansible + run: | + pip install "ansible>=9.0.0,<10.0.0" "ansible-core>=2.11,<2.17" + ansible --version + + # Configures Ansible to fail immediately on error, skip host key checking, use correct key file + - name: Write Ansible configuration file + run: | + cat > ~/.ansible.cfg << 'EOF' + [defaults] + any_errors_fatal = True + host_key_checking = False + max_fail_percentage = 0 + private_key_file = ~/.ssh/pet-east1.open.pem + EOF + + - name: Install this collection + run: ansible-galaxy collection install . --force + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }} + aws-region: us-east-1 + + - name: Setup SSH for GitLab + run: | + mkdir -p ~/.ssh + echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa + chmod 600 ~/.ssh/id_rsa + + - name: Setup SSH for EC2 + run: | + echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem + chmod 600 ~/.ssh/pet-east1.open.pem + + - name: Clone Themis repository + working-directory: .. + run: | + git clone git@gitlab.com:itential/platform-engineering/themis.git + cd themis + ls -la + + - name: Install Themis Python script dependencies + working-directory: ../themis + run: pip install -r scripts/requirements.txt + + # This is potentially more secure than adding the credentials as extra vars in the ansible-playbook command + - name: Add Nexus credentials to inventory + working-directory: ../themis + run: | + echo 'repository_username: "${{ secrets.NEXUS_USERNAME }}"' >> inventories/common/group_vars/all.yml + echo 'repository_password: "${{ secrets.NEXUS_PASSWORD }}"' >> inventories/common/group_vars/all.yml + + - name: Initialize OpenTofu + working-directory: ../themis/tofu_aws + run: tofu init + + # Copies the vars file for the selected design to the working directory to be imported automatically + - name: Set validated design + working-directory: ../themis/tofu_aws + run: cp tfvars/${{ inputs.design }}.tfvars design.auto.tfvars + + - name: Set additional job-specific variables + working-directory: ../themis/tofu_aws + run: | + cat > github_actions.auto.tfvars << 'EOF' + profile = "" + owner = "github" + os_type = "${{ inputs.os-type }}" + os_version = "${{ inputs.os-version }}" + EOF + + - name: Generate OpenTofu execution plan + working-directory: ../themis/tofu_aws + run: tofu plan -out=plan.tfplan + + - name: Provision EC2 instances + working-directory: ../themis/tofu_aws + run: tofu apply plan.tfplan + + - name: Generate Ansible inventory hosts file + working-directory: ../themis/tofu_aws + run: python3 ../scripts/generate_inventory.py --validate -o hosts.json + + # Retries until SSH connection is established or timeout is reached + - name: Wait for EC2 instances to be ready to SSH into + working-directory: ../themis + run: ansible all -m wait_for_connection -a "delay=10 timeout=300" -i tofu_aws/hosts.json -v + + # Waits for cloud init marker file to be written to disk (see cloud-init.tpl) + - name: Wait for cloud init script to complete + working-directory: ../themis + run: ansible all -m wait_for -a "path=/var/log/cloud-init-finished.marker timeout=300" -i tofu_aws/hosts.json -v + + - name: Run the deployer + working-directory: ../themis + run: > + ansible-playbook itential.deployer.site + -i tofu_aws/hosts.json + -i inventories/common + -i inventories/${{ inputs.design }} + -v + + - name: Verify that Platform is running correctly + working-directory: ../themis + run: | + for host in $(jq -r '.all.children.platform.hosts[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py platform "http://$host:3000" + done + for host in $(jq -r '(.all.children.platform_secondary.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py platform "http://$host:3000" + done + + - name: Verify that Gateway is running correctly + working-directory: ../themis + run: | + for host in $(jq -r '.all.children.gateway.hosts[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py gateway "http://$host:8083" + done + + - name: Verify that Redis is running correctly + working-directory: ../themis + run: | + for host in $(jq -r '.all.children.redis.hosts[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py redis "$host" + done + for host in $(jq -r '(.all.children.redis_secondary.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py redis "$host" + done + + - name: Verify that MongoDB is running correctly + working-directory: ../themis + run: | + for host in $(jq -r '.all.children.mongodb.hosts[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py mongodb "$host" + done + for host in $(jq -r '(.all.children.mongodb_arbiter.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do + python3 scripts/validate.py mongodb "$host" --arbiter + done + + - name: Terminate EC2 instances + if: always() + working-directory: ../themis/tofu_aws + run: tofu destroy -auto-approve diff --git a/.github/workflows/test-on-pull-request.yml b/.github/workflows/test-on-pull-request.yml new file mode 100644 index 00000000..0a56558e --- /dev/null +++ b/.github/workflows/test-on-pull-request.yml @@ -0,0 +1,20 @@ +name: Test Deployer on Pull Request + +on: + pull_request_target: + branches: + - main + +jobs: + run-test: + strategy: + matrix: + design: [aio, minimal, ha2, asa] + fail-fast: false + uses: ./.github/workflows/test-deployer.yml + with: + ref: ${{ github.event.pull_request.head.sha }} + design: ${{ matrix.design }} + os-type: rocky + os-version: "9" + secrets: inherit