diff --git a/.github/actions/install-tools/action.yml b/.github/actions/install-tools/action.yml index be487309d2c..41ba5d1acda 100644 --- a/.github/actions/install-tools/action.yml +++ b/.github/actions/install-tools/action.yml @@ -23,6 +23,7 @@ runs: uses: jdx/mise-action@146a28175021df8ca24f8ee1828cc2a60f980bd5 # v3.5.1 with: install_args: --env ci --jobs 1 + cache: ${{ runner.environment != 'self-hosted' }} env: MISE_VERBOSE: 1 GITHUB_TOKEN: ${{ inputs.token }} diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index dc90be0ee65..e663ee5427f 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -32,8 +32,12 @@ jobs: setup: needs: [optimize-ci] - if: needs.optimize-ci.outputs.skip == 'false' - runs-on: ubuntu-24.04 + if: | + needs.optimize-ci.outputs.skip == 'false' && ( + github.event_name == 'push' || + github.event.pull_request.head.repo.full_name == github.repository + ) + runs-on: [self-hosted, bench-lg] permissions: id-token: write outputs: @@ -75,8 +79,12 @@ jobs: strategy: matrix: ${{ fromJSON(needs.setup.outputs.unit) }} fail-fast: false - if: needs.setup.outputs.unit != '{"name":[],"include":[]}' - runs-on: ubuntu-24.04 + if: | + needs.setup.outputs.unit != '{"name":[],"include":[]}' && ( + github.event_name == 'push' || + github.event.pull_request.head.repo.full_name == github.repository + ) + runs-on: [self-hosted, bench-lg] steps: - name: Checkout base branch if: github.event_name == 'pull_request' @@ -84,10 +92,6 @@ jobs: with: ref: ${{ github.event.pull_request.base.sha }} - - name: Clean up disk - if: github.event_name == 'pull_request' - uses: ./.github/actions/clean-up-disk - - name: Install tools if: github.event_name == 'pull_request' uses: ./.github/actions/install-tools @@ -125,10 +129,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - - name: Clean up disk - if: github.event_name != 'pull_request' - uses: ./.github/actions/clean-up-disk - - name: Install tools uses: ./.github/actions/install-tools with: @@ -196,8 +196,12 @@ jobs: fail-fast: false env: HASH_GRAPH_PG_DATABASE: graph - if: needs.setup.outputs.integration != '{"name":[],"include":[]}' - runs-on: ubuntu-24.04 + if: | + needs.setup.outputs.integration != '{"name":[],"include":[]}' && ( + github.event_name == 'push' || + github.event.pull_request.head.repo.full_name == github.repository + ) + runs-on: [self-hosted, bench-lg] steps: - name: Checkout base branch if: github.event_name == 'pull_request' @@ -205,10 +209,6 @@ jobs: with: ref: ${{ github.event.pull_request.base.sha }} - - name: Clean up disk - if: github.event_name == 'pull_request' - uses: ./.github/actions/clean-up-disk - - name: Install tools if: github.event_name == 'pull_request' uses: ./.github/actions/install-tools @@ -238,8 +238,23 @@ jobs: touch .env.local echo 'HASH_GRAPH_PG_DATABASE=graph' > .env.local - cp .env.local .env.test.local + + # Tune Postgres for benchmarks + cat >> apps/hash-external-services/postgres/postgresql.conf << 'EOF' + # Benchmark optimizations + shared_buffers = 4GB + effective_cache_size = 8GB + work_mem = 256MB + maintenance_work_mem = 1GB + wal_buffers = 64MB + random_page_cost = 1.1 + effective_io_concurrency = 200 + synchronous_commit = off + EOF + + sed -i 's/shm_size: 1GB/shm_size: 4GB/' apps/hash-external-services/docker-compose.yml + yarn external-services:test up --wait - name: Start background tasks @@ -288,10 +303,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha }} - - name: Clean up disk - if: github.event_name != 'pull_request' - uses: ./.github/actions/clean-up-disk - - name: Install tools uses: ./.github/actions/install-tools with: @@ -318,8 +329,23 @@ jobs: touch .env.local echo 'HASH_GRAPH_PG_DATABASE=graph' > .env.local - cp .env.local .env.test.local + + # Tune Postgres for benchmarks + cat >> apps/hash-external-services/postgres/postgresql.conf << 'EOF' + # Benchmark optimizations + shared_buffers = 4GB + effective_cache_size = 8GB + work_mem = 256MB + maintenance_work_mem = 1GB + wal_buffers = 64MB + random_page_cost = 1.1 + effective_io_concurrency = 200 + synchronous_commit = off + EOF + + sed -i 's/shm_size: 1GB/shm_size: 4GB/' apps/hash-external-services/docker-compose.yml + yarn external-services:test up --wait - name: Start background tasks @@ -389,6 +415,10 @@ jobs: if: ${{ success() || failure() }} run: yarn workspace @apps/hash-external-services deploy logs --timestamps + - name: Tear down external services + if: ${{ success() || failure() }} + run: yarn workspace @apps/hash-external-services deploy:test down + - name: Upload background tasks logs uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: success() || failure() @@ -397,11 +427,21 @@ jobs: path: | var/logs + - name: Remove log files + if: ${{ success() || failure() }} + run: rm -rf var/logs + passed: name: Benches passed needs: [setup, unit-benches, integration-benches, optimize-ci] - if: always() && needs.optimize-ci.outputs.skip == 'false' - runs-on: ubuntu-latest + if: | + always() && + needs.optimize-ci.outputs.skip == 'false' && + ( + github.event_name == 'push' || + github.event.pull_request.head.repo.full_name == github.repository + ) + runs-on: ubuntu-24.04 permissions: pull-requests: write steps: diff --git a/apps/hash-external-services/docker-compose.bench.yml b/apps/hash-external-services/docker-compose.bench.yml new file mode 100644 index 00000000000..d3590854690 --- /dev/null +++ b/apps/hash-external-services/docker-compose.bench.yml @@ -0,0 +1,6 @@ +services: + postgres: + shm_size: 4GB + + volumes: + - ./postgres/postgresql.bench.conf:/etc/postgresql/postgresql.conf:ro diff --git a/apps/hash-external-services/docker-compose.test.yml b/apps/hash-external-services/docker-compose.test.yml index 4217d6a8dc9..e66899df4f5 100644 --- a/apps/hash-external-services/docker-compose.test.yml +++ b/apps/hash-external-services/docker-compose.test.yml @@ -48,7 +48,11 @@ services: condition: on-failure healthcheck: # Port 14269 is the Jaeger admin endpoint - test: [ "CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:14269 || exit 1" ] + test: + [ + "CMD-SHELL", + "wget --no-verbose --tries=1 --spider http://localhost:14269 || exit 1", + ] interval: 2s timeout: 2s retries: 10 diff --git a/apps/hash-external-services/package.json b/apps/hash-external-services/package.json index d87b7d474b7..b9a14d8e752 100644 --- a/apps/hash-external-services/package.json +++ b/apps/hash-external-services/package.json @@ -12,6 +12,7 @@ "build:temporal:setup": "docker buildx build --file temporal/setup.Dockerfile --build-arg TEMPORAL_VERSION=1.23.1.0 temporal --load", "codegen": "touch ../../.env.local", "deploy": "docker compose --project-name hash-external-services --file docker-compose.yml --file docker-compose.dev.yml --env-file ../../.env --env-file ../../.env.development --env-file ../../.env.local", + "deploy:bench": "docker compose --project-name hash-external-services --file docker-compose.yml --file docker-compose.test.yml --file docker-compose.bench.yml --env-file ../../.env --env-file ../../.env.test --env-file ../../.env.local", "deploy:prod": "docker compose --project-name hash-external-services --file docker-compose.yml --file docker-compose.prod.yml --env-file ../../.env --env-file ../../.env.production --env-file ../../.env.local", "deploy:test": "docker compose --project-name hash-external-services --file docker-compose.yml --file docker-compose.test.yml --env-file ../../.env --env-file ../../.env.test --env-file ../../.env.local" } diff --git a/apps/hash-external-services/postgres/postgresql.bench.conf b/apps/hash-external-services/postgres/postgresql.bench.conf new file mode 100644 index 00000000000..19ea8f9e31d --- /dev/null +++ b/apps/hash-external-services/postgres/postgresql.bench.conf @@ -0,0 +1,14 @@ +# Allow connections from all addresses +listen_addresses = '*' +log_min_messages = warning + +max_connections = 300 + +shared_buffers = 4GB +effective_cache_size = 8GB +work_mem = 256MB +maintenance_work_mem = 1GB +wal_buffers = 64MB +random_page_cost = 1.1 +effective_io_concurrency = 200 +synchronous_commit = off diff --git a/package.json b/package.json index 75c139fc80c..466fca86c5f 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "dev:backend:api": "CARGO_TERM_PROGRESS_WHEN=never turbo dev --log-order stream --filter '@apps/hash-api' --", "dev:frontend": "CARGO_TERM_PROGRESS_WHEN=never turbo dev --log-order stream --filter '@apps/hash-frontend' --", "external-services": "turbo deploy --filter '@apps/hash-external-services' --", + "external-services:bench": "turbo deploy:bench --filter '@apps/hash-external-services' --", "external-services:offline": "turbo deploy:offline --filter '@apps/hash-external-services' --", "external-services:prod": "turbo deploy:prod --filter '@apps/hash-external-services' --", "external-services:test": "turbo deploy:test --filter '@apps/hash-external-services' --",