Add deployer test pipeline

.github/workflows/test-deployer.yml

@@ -0,0 +1,234 @@
+# Runs the Itential Deployer on EC2 instances conforming to some validated design and other parameters 
+# (e.g. OS type and version), and validates that the deployed services are all running correctly.
+# This allows for testing deployments on various configurations in parallel in a fully automated manner.
+
+# Main steps performed by this workflow:
+# - Cloning Themis from GitLab
+# - Provisioning EC2 instances using Themis
+# - Converting OpenTofu output to Ansible inventory
+# - Running deployer on newly created instances
+# - Running validation script to test each service (Redis, MongoDB, Platform, Gateway)
+# - Terminating all instances using Themis, irrespective of success
+
+
+name: Test Deployer
+
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        required: false
+        type: string
+      design:
+        required: true
+        type: string
+      os-type:
+        required: true
+        type: string
+      os-version:
+        required: true
+        type: string
+    secrets:
+      AWS_ACCESS_KEY_ID:
+        required: true
+      AWS_SECRET_ACCESS_KEY:
+        required: true
+      AWS_SESSION_TOKEN:
+        required: true
+      GITLAB_SSH_KEY:
+        required: true
+      EC2_SSH_KEY:
+        required: true
+      NEXUS_USERNAME:
+        required: true
+      NEXUS_PASSWORD:
+        required: true
+
+
+jobs:
+  test-deployer:
+    runs-on: self-hosted
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-latest
+    env:
+      GIT_SSH_COMMAND: ssh -i ~/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
+      PIP_BREAK_SYSTEM_PACKAGES: "1"
+    timeout-minutes: 60
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.ref }}
+
+    - name: Update package lists
+      run: sudo apt update
+
+    - name: Install Python
+      run: |
+        sudo apt install -y python3
+        wget https://bootstrap.pypa.io/get-pip.py && sudo python3 get-pip.py && rm get-pip.py
+        sudo ln -s /usr/bin/python3 /usr/bin/python || true
+        sudo ln -s /usr/bin/pip3 /usr/bin/pip || true
+        python --version
+        pip --version
+
+    # For some reason, Node appears to be a dependency of Themis
+    - name: Install Node.js
+      run: |
+        sudo apt install -y nodejs
+        node --version
+
+    - name: Install OpenTofu
+      uses: opentofu/setup-opentofu@v1
+      with:
+        tofu_version: latest
+
+    - name: Install Ansible
+      run: |
+        pip install "ansible>=9.0.0,<10.0.0" "ansible-core>=2.11,<2.17"
+        ansible --version
+
+    # Configures Ansible to fail immediately on error, skip host key checking, use correct key file
+    - name: Write Ansible configuration file
+      run: |
+        cat > ~/.ansible.cfg << 'EOF'
+        [defaults]
+        any_errors_fatal = True
+        host_key_checking = False
+        max_fail_percentage = 0
+        private_key_file = ~/.ssh/pet-east1.open.pem
+        EOF
+
+    - name: Install this collection
+      run: ansible-galaxy collection install . --force
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-session-token: ${{ secrets.AWS_SESSION_TOKEN }}
+        aws-region: us-east-1
+
+    - name: Setup SSH for GitLab
+      run: |
+        mkdir -p ~/.ssh
+        echo "${{ secrets.GITLAB_SSH_KEY }}" > ~/.ssh/id_rsa
+        chmod 600 ~/.ssh/id_rsa
+
+    - name: Setup SSH for EC2
+      run: |
+        echo "${{ secrets.EC2_SSH_KEY }}" > ~/.ssh/pet-east1.open.pem
+        chmod 600 ~/.ssh/pet-east1.open.pem
+
+    - name: Clone Themis repository
+      working-directory: ..
+      run: |
+        git clone git@gitlab.com:itential/platform-engineering/themis.git
+        cd themis
+        ls -la
+
+    - name: Install Themis Python script dependencies
+      working-directory: ../themis
+      run: pip install -r scripts/requirements.txt
+
+    # This is potentially more secure than adding the credentials as extra vars in the ansible-playbook command
+    - name: Add Nexus credentials to inventory
+      working-directory: ../themis
+      run: |
+        echo 'repository_username: "${{ secrets.NEXUS_USERNAME }}"' >> inventories/common/group_vars/all.yml
+        echo 'repository_password: "${{ secrets.NEXUS_PASSWORD }}"' >> inventories/common/group_vars/all.yml
+
+    - name: Initialize OpenTofu
+      working-directory: ../themis/tofu_aws
+      run: tofu init
+
+    # Copies the vars file for the selected design to the working directory to be imported automatically
+    - name: Set validated design
+      working-directory: ../themis/tofu_aws
+      run: cp tfvars/${{ inputs.design }}.tfvars design.auto.tfvars
+
+    - name: Set additional job-specific variables
+      working-directory: ../themis/tofu_aws
+      run: |
+        cat > github_actions.auto.tfvars << 'EOF'
+        profile = ""
+        owner = "github"
+        os_type = "${{ inputs.os-type }}"
+        os_version = "${{ inputs.os-version }}"
+        EOF
+
+    - name: Generate OpenTofu execution plan
+      working-directory: ../themis/tofu_aws
+      run: tofu plan -out=plan.tfplan
+
+    - name: Provision EC2 instances
+      working-directory: ../themis/tofu_aws
+      run: tofu apply plan.tfplan
+
+    - name: Generate Ansible inventory hosts file
+      working-directory: ../themis/tofu_aws
+      run: python3 ../scripts/generate_inventory.py --validate -o hosts.json
+
+    # Retries until SSH connection is established or timeout is reached
+    - name: Wait for EC2 instances to be ready to SSH into
+      working-directory: ../themis
+      run: ansible all -m wait_for_connection -a "delay=10 timeout=300" -i tofu_aws/hosts.json -v
+
+    # Waits for cloud init marker file to be written to disk (see cloud-init.tpl)
+    - name: Wait for cloud init script to complete
+      working-directory: ../themis
+      run: ansible all -m wait_for -a "path=/var/log/cloud-init-finished.marker timeout=300" -i tofu_aws/hosts.json -v
+
+    - name: Run the deployer
+      working-directory: ../themis
+      run: >
+        ansible-playbook itential.deployer.site 
+        -i tofu_aws/hosts.json 
+        -i inventories/common 
+        -i inventories/${{ inputs.design }} 
+        -v
+
+    - name: Verify that Platform is running correctly
+      working-directory: ../themis
+      run: |
+        for host in $(jq -r '.all.children.platform.hosts[] | .ansible_host' tofu_aws/hosts.json); do
+          python3 scripts/validate.py platform "http://$host:3000"
+        done
+        for host in $(jq -r '(.all.children.platform_secondary.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do
+          python3 scripts/validate.py platform "http://$host:3000"
+        done
+
+    - name: Verify that Gateway is running correctly
+      working-directory: ../themis
+      run: |
+        for host in $(jq -r '.all.children.gateway.hosts[] | .ansible_host' tofu_aws/hosts.json); do
+          python3 scripts/validate.py gateway "http://$host:8083"
+        done
+
+    - name: Verify that Redis is running correctly
+      working-directory: ../themis
+      run: |
+        for host in $(jq -r '.all.children.redis.hosts[] | .ansible_host' tofu_aws/hosts.json); do
+          python3 scripts/validate.py redis "$host"
+        done
+        for host in $(jq -r '(.all.children.redis_secondary.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do
+          python3 scripts/validate.py redis "$host"
+        done
+
+    - name: Verify that MongoDB is running correctly
+      working-directory: ../themis
+      run: |
+        for host in $(jq -r '.all.children.mongodb.hosts[] | .ansible_host' tofu_aws/hosts.json); do
+          python3 scripts/validate.py mongodb "$host"
+        done
+        for host in $(jq -r '(.all.children.mongodb_arbiter.hosts // [])[] | .ansible_host' tofu_aws/hosts.json); do
+          python3 scripts/validate.py mongodb "$host" --arbiter
+        done
+
+    - name: Terminate EC2 instances
+      if: always()
+      working-directory: ../themis/tofu_aws
+      run: tofu destroy -auto-approve

.github/workflows/test-on-pull-request.yml

@@ -0,0 +1,20 @@
+name: Test Deployer on Pull Request
+
+on:
+  pull_request_target:
+    branches:
+      - main
+
+jobs:
+  run-test:
+    strategy:
+      matrix:
+        design: [aio, minimal, ha2, asa]
+      fail-fast: false
+    uses: ./.github/workflows/test-deployer.yml
+    with:
+      ref: ${{ github.event.pull_request.head.sha }}
+      design: ${{ matrix.design }}
+      os-type: rocky
+      os-version: "9"
+    secrets: inherit