SRE-231: Enable self-hosted benchmark runners

.github/actions/install-tools/action.yml

@@ -23,6 +23,7 @@ runs:
       uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3.5.1
       with:
         install_args: --env ci --jobs 1
+        cache: ${{ runner.environment != 'self-hosted' }}
       env:
         MISE_VERBOSE: 1
         GITHUB_TOKEN: ${{ inputs.token }}

.github/workflows/bench.yml

@@ -32,8 +32,12 @@ jobs:
 
   setup:
     needs: [optimize-ci]
-    if: needs.optimize-ci.outputs.skip == 'false'
-    runs-on: ubuntu-24.04
+    if: |
+      needs.optimize-ci.outputs.skip == 'false' && (
+        github.event_name == 'push' ||
+        github.event.pull_request.head.repo.full_name == github.repository
+      )
+    runs-on: [self-hosted, bench-lg]
     permissions:
       id-token: write
     outputs:
@@ -75,19 +79,19 @@ jobs:
     strategy:
       matrix: ${{ fromJSON(needs.setup.outputs.unit) }}
       fail-fast: false
-    if: needs.setup.outputs.unit != '{"name":[],"include":[]}'
-    runs-on: ubuntu-24.04
+    if: |
+      needs.setup.outputs.unit != '{"name":[],"include":[]}' && (
+        github.event_name == 'push' ||
+        github.event.pull_request.head.repo.full_name == github.repository
+      )
+    runs-on: [self-hosted, bench-lg]
     steps:
       - name: Checkout base branch
         if: github.event_name == 'pull_request'
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           ref: ${{ github.event.pull_request.base.sha }}
 
-      - name: Clean up disk
-        if: github.event_name == 'pull_request'
-        uses: ./.github/actions/clean-up-disk
-
       - name: Install tools
         if: github.event_name == 'pull_request'
         uses: ./.github/actions/install-tools
@@ -125,10 +129,6 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Clean up disk
-        if: github.event_name != 'pull_request'
-        uses: ./.github/actions/clean-up-disk
-
       - name: Install tools
         uses: ./.github/actions/install-tools
         with:
@@ -196,19 +196,19 @@ jobs:
       fail-fast: false
     env:
       HASH_GRAPH_PG_DATABASE: graph
-    if: needs.setup.outputs.integration != '{"name":[],"include":[]}'
-    runs-on: ubuntu-24.04
+    if: |
+      needs.setup.outputs.integration != '{"name":[],"include":[]}' && (
+        github.event_name == 'push' ||
+        github.event.pull_request.head.repo.full_name == github.repository
+      )
+    runs-on: [self-hosted, bench-lg]
     steps:
       - name: Checkout base branch
         if: github.event_name == 'pull_request'
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
         with:
           ref: ${{ github.event.pull_request.base.sha }}
 
-      - name: Clean up disk
-        if: github.event_name == 'pull_request'
-        uses: ./.github/actions/clean-up-disk
-
       - name: Install tools
         if: github.event_name == 'pull_request'
         uses: ./.github/actions/install-tools
@@ -238,8 +238,23 @@ jobs:
           touch .env.local
 
           echo 'HASH_GRAPH_PG_DATABASE=graph' > .env.local
-
           cp .env.local .env.test.local
+
+          # Tune Postgres for benchmarks
+          cat >> apps/hash-external-services/postgres/postgresql.conf << 'EOF'
+          # Benchmark optimizations
+          shared_buffers = 4GB
+          effective_cache_size = 8GB
+          work_mem = 256MB
+          maintenance_work_mem = 1GB
+          wal_buffers = 64MB
+          random_page_cost = 1.1
+          effective_io_concurrency = 200
+          synchronous_commit = off
+          EOF
+
+          sed -i 's/shm_size: 1GB/shm_size: 4GB/' apps/hash-external-services/docker-compose.yml
+
           yarn external-services:test up --wait
 
       - name: Start background tasks
@@ -288,10 +303,6 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
 
-      - name: Clean up disk
-        if: github.event_name != 'pull_request'
-        uses: ./.github/actions/clean-up-disk
-
       - name: Install tools
         uses: ./.github/actions/install-tools
         with:
@@ -318,8 +329,23 @@ jobs:
           touch .env.local
 
           echo 'HASH_GRAPH_PG_DATABASE=graph' > .env.local
-
           cp .env.local .env.test.local
+
+          # Tune Postgres for benchmarks
+          cat >> apps/hash-external-services/postgres/postgresql.conf << 'EOF'
+          # Benchmark optimizations
+          shared_buffers = 4GB
+          effective_cache_size = 8GB
+          work_mem = 256MB
+          maintenance_work_mem = 1GB
+          wal_buffers = 64MB
+          random_page_cost = 1.1
+          effective_io_concurrency = 200
+          synchronous_commit = off
+          EOF
+
+          sed -i 's/shm_size: 1GB/shm_size: 4GB/' apps/hash-external-services/docker-compose.yml
+
           yarn external-services:test up --wait
 
       - name: Start background tasks
@@ -389,6 +415,10 @@ jobs:
         if: ${{ success() || failure() }}
         run: yarn workspace @apps/hash-external-services deploy logs --timestamps
 
+      - name: Tear down external services
+        if: ${{ success() || failure() }}
+        run: yarn workspace @apps/hash-external-services deploy:test down
+
       - name: Upload background tasks logs
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
         if: success() || failure()
@@ -397,11 +427,21 @@ jobs:
           path: |
             var/logs
 
+      - name: Remove log files
+        if: ${{ success() || failure() }}
+        run: rm -rf var/logs
+
   passed:
     name: Benches passed
     needs: [setup, unit-benches, integration-benches, optimize-ci]
-    if: always() && needs.optimize-ci.outputs.skip == 'false'
-    runs-on: ubuntu-latest
+    if: |
+      always() &&
+      needs.optimize-ci.outputs.skip == 'false' &&
+      (
+        github.event_name == 'push' ||
+        github.event.pull_request.head.repo.full_name == github.repository
+      )
+    runs-on: ubuntu-24.04
     permissions:
       pull-requests: write
     steps:

apps/hash-external-services/docker-compose.bench.yml

@@ -0,0 +1,6 @@
+services:
+  postgres:
+    shm_size: 4GB
+
+    volumes:
+      - ./postgres/postgresql.bench.conf:/etc/postgresql/postgresql.conf:ro

apps/hash-external-services/docker-compose.test.yml

@@ -48,7 +48,11 @@ services:
         condition: on-failure
     healthcheck:
       # Port 14269 is the Jaeger admin endpoint
-      test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:14269 || exit 1" ]
+      test:
+        [
+          "CMD-SHELL",
+          "wget --no-verbose --tries=1 --spider http://localhost:14269 || exit 1",
+        ]
       interval: 2s
       timeout: 2s
       retries: 10

apps/hash-external-services/package.json

@@ -12,6 +12,7 @@
     "build:temporal:setup": "docker buildx build --file temporal/setup.Dockerfile --build-arg TEMPORAL_VERSION=1.23.1.0 temporal --load",
     "codegen": "touch ../../.env.local",
     "deploy": "docker compose --project-name hash-external-services --file docker-compose.yml --file docker-compose.dev.yml --env-file ../../.env --env-file ../../.env.development --env-file ../../.env.local",
+    "deploy:bench": "docker compose --project-name hash-external-services --file docker-compose.yml --file docker-compose.test.yml --file docker-compose.bench.yml --env-file ../../.env --env-file ../../.env.test --env-file ../../.env.local",
     "deploy:prod": "docker compose --project-name hash-external-services --file docker-compose.yml --file docker-compose.prod.yml --env-file ../../.env --env-file ../../.env.production --env-file ../../.env.local",
     "deploy:test": "docker compose --project-name hash-external-services --file docker-compose.yml --file docker-compose.test.yml --env-file ../../.env --env-file ../../.env.test --env-file ../../.env.local"
   }

apps/hash-external-services/postgres/postgresql.bench.conf

@@ -0,0 +1,14 @@
+# Allow connections from all addresses
+listen_addresses = '*'
+log_min_messages = warning
+
+max_connections = 300
+
+shared_buffers = 4GB
+effective_cache_size = 8GB
+work_mem = 256MB
+maintenance_work_mem = 1GB
+wal_buffers = 64MB
+random_page_cost = 1.1
+effective_io_concurrency = 200
+synchronous_commit = off

package.json

@@ -28,6 +28,7 @@
     "dev:backend:api": "CARGO_TERM_PROGRESS_WHEN=never turbo dev --log-order stream --filter '@apps/hash-api' --",
     "dev:frontend": "CARGO_TERM_PROGRESS_WHEN=never turbo dev --log-order stream --filter '@apps/hash-frontend' --",
     "external-services": "turbo deploy --filter '@apps/hash-external-services' --",
+    "external-services:bench": "turbo deploy:bench --filter '@apps/hash-external-services' --",
     "external-services:offline": "turbo deploy:offline --filter '@apps/hash-external-services' --",
     "external-services:prod": "turbo deploy:prod --filter '@apps/hash-external-services' --",
     "external-services:test": "turbo deploy:test --filter '@apps/hash-external-services' --",