diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a1976a223e..0c88a645e91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -389,7 +389,10 @@ jobs: cardano_node_version: ["10.4.1", "10.5.1"] hard_fork_latest_era_at_epoch: [0] run_id: ["#1", "#2", "#3"] - extra_args: ["--aggregate-signature-type=Concatenation"] + extra_args: + [ + "--aggregate-signature-type=Concatenation --use-dmq --dmq-node-flavor=haskell", + ] include: # Include a test for partial decentralization with leader/follower signer registration and P2P signature registration with fake DMQ node @@ -400,7 +403,15 @@ jobs: hard_fork_latest_era_at_epoch: 0 run_id: "#1" extra_args: "--number-of-aggregators=2 --use-relays --relay-signer-registration-mode=passthrough --relay-signature-registration-mode=p2p --aggregate-signature-type=Concatenation --use-dmq --dmq-node-flavor=fake" - # Include a test for partial decentralization with leader/follower signer registration and P2P signature registration + # Include a test for partial decentralization with leader/follower signer registration and P2P signature registration with fake DMQ node + - mode: "leader-follower" + era: ${{ fromJSON(needs.build-ubuntu.outputs.eras)[0] }} + next_era: [""] + cardano_node_version: "10.5.1" + hard_fork_latest_era_at_epoch: 0 + run_id: "#1" + extra_args: "--number-of-aggregators=2 --use-relays --relay-signer-registration-mode=passthrough --relay-signature-registration-mode=p2p --aggregate-signature-type=Concatenation --use-dmq --dmq-node-flavor=fake" + # Include a test for partial decentralization with leader/follower signer registration and P2P signature registration with Mithril relay - mode: "leader-follower" era: ${{ fromJSON(needs.build-ubuntu.outputs.eras)[0] }} next_era: [""] diff --git a/.github/workflows/test-deploy-network.yml b/.github/workflows/test-deploy-network.yml index 04996133bb9..819da5744f9 100644 --- a/.github/workflows/test-deploy-network.yml +++ b/.github/workflows/test-deploy-network.yml @@ -89,7 +89,7 @@ jobs: { "1" = { type = "unverified-cardano-passive-norelay", - pool_id = "pool1y0uxkqyplyx6ld25e976t0s35va3ysqcscatwvy2sd2cwcareq7", + pool_id = "pool13zafxlpfgymf474uv52qt557z5k5frn9p83yr55zp267wj5mpu4", }, } mithril_leader_aggregator_endpoint: https://aggregator.dev-preview.api.mithril.network/aggregator diff --git a/docs/website/root/manual/operate/run-signer-node.md b/docs/website/root/manual/operate/run-signer-node.md index 15c3af117e6..53fec1e63cd 100644 --- a/docs/website/root/manual/operate/run-signer-node.md +++ b/docs/website/root/manual/operate/run-signer-node.md @@ -99,10 +99,12 @@ Note that this guide works on a Linux machine only. ::: - To operate a **Cardano node** as a **stake pool**, you need: + - The pool's `operational certificate` - The pool's `KES secret key` - To access the file system of the **Cardano block producer** node for **production** deployment (or of the **Cardano relay** node for **naive** deployment), you will need the following permissions: + - Read rights on the `Database` folder (specified by the `--database-path` setting of the **Cardano node**) - Read and write rights on the `Inter Process Communication` file (typically defined by the `CARDANO_NODE_SOCKET_PATH` environment variable used to launch the **Cardano node**) @@ -546,6 +548,7 @@ sudo systemctl restart mithril-signer :::info - If you have already installed `Squid` via `apt` package manager, we recommend that you delete it before manually building it from source by running the following commands: + - `sudo systemctl stop squid` - `sudo apt remove squid` - `sudo apt autoremove`. @@ -889,6 +892,53 @@ sudo service netfilter-persistent save ::: +## Setup the DMQ node (unstable) + +:::caution + +The DMQ node setup is currently **unstable** and not suitable for production use. + +::: + +The DMQ node supports the implementation of a **Decentralized Message Queue** (DMQ) for Mithril. The DMQ protocol allows Mithril signers to exchange signatures in a decentralized manner, enhancing the robustness and scalability of the Mithril networks. Once stabilized and deployed on a majority of SPOs on a Mithril network, the DMQ protocol will allow multiple aggregators to operate simultaneously, improving the overall availability of the Mithril protocol. The DMQ protocol is fully described in the [CIP-0137](https://cips.cardano.org/cip/CIP-0137). + +### Download the pre-built binary + +:::tip + +You can use these parameters for the **pre-release-preview** network: + +- **DMQ_RELEASE_URL**: `https://github.com/input-output-hk/mithril/raw/refs/heads/jpraynaud/dmq-node-binary/mithril-test-lab/mithril-devnet/bin/dmq-node-0.2.0.0-53bf9652787dc768abd86cf3844f1206f0fd7d8c` + +This URL may change in the future; please refer to this page for the latest released version of the DMQ node binary. + +::: + +To download the latest released version of the DMQ node binary, run the following command: + +```bash +curl --fail -sL -o dmq-node **DMQ_RELEASE_URL** +``` + +### Installing the service + +#### Make the binary executable + +To make the binary executable, run: + +```bash +chmod +x dmq-node +``` + +#### Move the executable + +To move the executable to /opt/mithril, run: + +```bash +sudo mkdir -p /opt/mithril +sudo mv dmq-node /opt/mithril +``` + ## Verify the Mithril signer deployment :::tip diff --git a/mithril-common/Cargo.toml b/mithril-common/Cargo.toml index a804954dc5e..04b63d448c2 100644 --- a/mithril-common/Cargo.toml +++ b/mithril-common/Cargo.toml @@ -19,7 +19,7 @@ rustdoc-args = ["--cfg", "docsrs"] ignored = ["serde_bytes"] [features] -default = ["rug-backend"] +default = ["rug-backend", "allow_skip_signer_certification"] # Enables `rug-backend` features for `mithril-stm` dependency rug-backend = ["mithril-stm/rug-backend"] diff --git a/mithril-infra/assets/docker/Dockerfile.dmq b/mithril-infra/assets/docker/Dockerfile.dmq new file mode 100644 index 00000000000..40028505275 --- /dev/null +++ b/mithril-infra/assets/docker/Dockerfile.dmq @@ -0,0 +1,11 @@ +FROM alpine:latest +ARG DMQ_NODE_BINARY_URL +ARG MITHRIL_IMAGE_ID # Only for triggering rebuilds when mithril image changes + +RUN apk add --no-cache wget + +RUN wget -O /bin/dmq-node $DMQ_NODE_BINARY_URL +RUN chmod +x /bin/dmq-node +RUN /bin/dmq-node --version + +ENTRYPOINT ["/bin/dmq-node"] \ No newline at end of file diff --git a/mithril-infra/assets/docker/dmq/config/config.json b/mithril-infra/assets/docker/dmq/config/config.json new file mode 100644 index 00000000000..3b3ba70f8fd --- /dev/null +++ b/mithril-infra/assets/docker/dmq/config/config.json @@ -0,0 +1,17 @@ +{ + "LocalMsgSubmissionTracer": true, + "LocalMsgNotificationTracer": true, + "ConnectionManagerTracer": true, + "DiffusionTracer": false, + "InboundGovernorTracer": false, + "LocalInboundGovernorTracer": false, + "PeerSelectionTracer": false, + "PeerSelectionCounters": false, + "PeerSharing": false, + "SigSubmissionLogicTracer": false, + "SigSubmissionClientTracer": false, + "SigSubmissionServerTracer": false, + "MuxTracer": false, + "ChannelTracer": false, + "DebugPeerSelectionTracer": false +} diff --git a/mithril-infra/assets/docker/dmq/config/topology.json b/mithril-infra/assets/docker/dmq/config/topology.json new file mode 100644 index 00000000000..59645255b6c --- /dev/null +++ b/mithril-infra/assets/docker/dmq/config/topology.json @@ -0,0 +1,18 @@ +{ + "bootstrapPeers": [], + "localRoots": [ + { + "accessPoints": [], + "advertise": false, + "trustable": false, + "valency": 2 + } + ], + "peerSnapshotFile": null, + "publicRoots": [ + { + "accessPoints": [], + "advertise": false + } + ] +} diff --git a/mithril-infra/assets/docker/docker-compose-aggregator-p2p-dmq-override.yaml b/mithril-infra/assets/docker/docker-compose-aggregator-p2p-dmq-fake-node-override.yaml similarity index 100% rename from mithril-infra/assets/docker/docker-compose-aggregator-p2p-dmq-override.yaml rename to mithril-infra/assets/docker/docker-compose-aggregator-p2p-dmq-fake-node-override.yaml diff --git a/mithril-infra/assets/docker/docker-compose-aggregator-p2p-dmq-real-node-override.yaml b/mithril-infra/assets/docker/docker-compose-aggregator-p2p-dmq-real-node-override.yaml new file mode 100644 index 00000000000..a53073145ac --- /dev/null +++ b/mithril-infra/assets/docker/docker-compose-aggregator-p2p-dmq-real-node-override.yaml @@ -0,0 +1,52 @@ +services: + mithril-aggregator: + volumes: + - ../data/${NETWORK}/mithril-aggregator/dmq/ipc:/ipc-dmq + environment: + - DMQ_NODE_SOCKET_PATH=/ipc-dmq/dmq.socket + - NETWORK=${NETWORK} + - NETWORK_MAGIC=${NETWORK_MAGIC} + + dmq-node: + # TODO: gather logs and send to loki + image: dmq-node-${MITHRIL_IMAGE_ID} + container_name: dmq-node-aggregator + restart: always + user: ${CURRENT_UID} + profiles: + - dmq + - all + build: + context: . + dockerfile: Dockerfile.dmq + args: + DMQ_NODE_BINARY_URL: ${DMQ_NODE_BINARY_URL} + MITHRIL_IMAGE_ID: ${MITHRIL_IMAGE_ID} + volumes: + - ../data/${NETWORK}/mithril-aggregator/dmq/config:/config + - ../data/${NETWORK}/mithril-aggregator/dmq/ipc:/ipc + - ../data/${NETWORK}/mithril-aggregator/cardano/ipc:/ipc-cardano + ports: + - "${AGGREGATOR_DMQ_PORT}:${AGGREGATOR_DMQ_PORT}" + logging: + driver: "${LOGGING_DRIVER}" + options: + tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" + command: + [ + "--configuration-file", + "/config/config.json", + "--topology-file", + "/config/topology.json", + "--local-socket", + "/ipc/dmq.socket", + "--host-addr", + "${AGGREGATOR_DMQ_ADDR}", + "--port", + "${AGGREGATOR_DMQ_PORT}", + ] + +networks: + default: + name: mithril_network + external: true diff --git a/mithril-infra/assets/docker/docker-compose-reverse-proxy.yaml b/mithril-infra/assets/docker/docker-compose-reverse-proxy.yaml index d372dcb9176..caa9f5edb74 100644 --- a/mithril-infra/assets/docker/docker-compose-reverse-proxy.yaml +++ b/mithril-infra/assets/docker/docker-compose-reverse-proxy.yaml @@ -1,6 +1,6 @@ services: reverse-proxy: - image: traefik:3.3 + image: traefik:3.6 container_name: reverse-proxy restart: always user: "${CURRENT_UID}:${DOCKER_GID}" diff --git a/mithril-infra/assets/docker/docker-compose-signer-p2p-base-override.yaml b/mithril-infra/assets/docker/docker-compose-signer-p2p-base-no-dmq-override.yaml similarity index 100% rename from mithril-infra/assets/docker/docker-compose-signer-p2p-base-override.yaml rename to mithril-infra/assets/docker/docker-compose-signer-p2p-base-no-dmq-override.yaml diff --git a/mithril-infra/assets/docker/docker-compose-signer-p2p-bootstrap-override.yaml b/mithril-infra/assets/docker/docker-compose-signer-p2p-bootstrap-no-dmq-override.yaml similarity index 100% rename from mithril-infra/assets/docker/docker-compose-signer-p2p-bootstrap-override.yaml rename to mithril-infra/assets/docker/docker-compose-signer-p2p-bootstrap-no-dmq-override.yaml diff --git a/mithril-infra/assets/docker/docker-compose-signer-p2p-dmq-override.yaml b/mithril-infra/assets/docker/docker-compose-signer-p2p-dmq-fake-node-override.yaml similarity index 100% rename from mithril-infra/assets/docker/docker-compose-signer-p2p-dmq-override.yaml rename to mithril-infra/assets/docker/docker-compose-signer-p2p-dmq-fake-node-override.yaml diff --git a/mithril-infra/assets/docker/docker-compose-signer-p2p-dmq-real-node-override.yaml b/mithril-infra/assets/docker/docker-compose-signer-p2p-dmq-real-node-override.yaml new file mode 100644 index 00000000000..e30a3d8d6e1 --- /dev/null +++ b/mithril-infra/assets/docker/docker-compose-signer-p2p-dmq-real-node-override.yaml @@ -0,0 +1,50 @@ +services: + mithril-signer: + volumes: + - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/dmq/ipc:/ipc-dmq + environment: + - DMQ_NODE_SOCKET_PATH=/ipc-dmq/dmq.socket + - NETWORK_MAGIC=${NETWORK_MAGIC} + - SIGNATURE_PUBLISHER_SKIP_DELAYER=true # TODO: remove + dmq-node: + image: dmq-node-${MITHRIL_IMAGE_ID} + container_name: dmq-node-signer-${SIGNER_ID} + restart: always + user: ${CURRENT_UID} + profiles: + - dmq + - all + build: + context: . + dockerfile: Dockerfile.dmq + args: + DMQ_NODE_BINARY_URL: ${DMQ_NODE_BINARY_URL} + MITHRIL_IMAGE_ID: ${MITHRIL_IMAGE_ID} + volumes: + - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/dmq/config:/config + - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/dmq/ipc:/ipc + - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/ipc:/ipc-cardano + ports: + - "${SIGNER_DMQ_PORT}:${SIGNER_DMQ_PORT}" + logging: + driver: "${LOGGING_DRIVER}" + options: + tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}" + command: + [ + "--configuration-file", + "/config/config.json", + "--topology-file", + "/config/topology.json", + "--local-socket", + "/ipc/dmq.socket", + "--host-addr", + "${SIGNER_DMQ_ADDR}", + "--port", + "${SIGNER_DMQ_PORT}", + ] + +networks: + default: + name: mithril_network + external: true diff --git a/mithril-infra/assets/ssh_keys-testing b/mithril-infra/assets/ssh_keys-testing index ba2a790a0db..c14f0e8d7e0 100644 --- a/mithril-infra/assets/ssh_keys-testing +++ b/mithril-infra/assets/ssh_keys-testing @@ -2,3 +2,4 @@ curry:ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3bO4CpX+S2y1X9Q2v7L9sPV5ZiGgo3B4rv4k root:ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3bO4CpX+S2y1X9Q2v7L9sPV5ZiGgo3B4rv4k3pgmko jpraynaud curry:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDkI86FAS+Bdce+gr0BpfyijTmrX+I/smi8xOsPVSrxFwQbo5HYwbuaLzQ2RfxEvb+rwaVyeIMLNGe5jWTYc/+tYTV40aQ8PkrUTcl7olazkelHbkgxq1ScN/GQvu7xDg+vjiH5vO7AZOn/NWq9CKeSB8rb/kZvcX1uWP+NygKAthYo33AFw9XoEhO/GUVSgG1M8ZRg5oSdJYZNvUiEGd/phe5OlwWH/4IcPOuyR4M9FAMAv9WSBzcAfQNO0SHJjBMFUkGAfw1aTqd0GPAfrRBLkJ2XyBRBkDKp72WVmPIdwDihoFIo1rOYvw0qQby//vbKpbTrL2cItNbqTwO2UY3FVkZ5MhJp9QaUUr/Ocie+KbE3jFeiVirIqxPksR+O19xAevKz2XL8RlICuZSBDovhjqBzvow36OdCm5mdCLX0psOSeIAS4I0iZaSooo+z1q4aXM+wYMgJxUF2IHr7TdGbHThtsDysPVuVlCai2+c/XC7IwIK1wrm+y5W0wPu6Vu0= alenar curry:ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBce8iKP5p4FI1+vLvr1R7NtIcwwyIEh4bRC1fyI+6oT turmelclem +curry:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNGKX8fIDYbnc0eBY+IjD6wBMEKHjCNeWKeCAmt+KbGlF/r2Z4POLIZb4eUNs/P9BQn1d85V8dR64TpM05Af7n9bpQFN5+MOgFD7/o78gGJXKHrgnXemuMh7Q3LMqjgDIb+WixDDYDq1c68u03QM6G/7qYCotjoE/pQI0o7143RSnkQQGxFSzZ9WTgrgvKJK+6ENpC6ItGLm4rZAzdyez/3sOkT8JO+3Da8xFJVf7Ladf91lpwONAIvGCROG7bZl0NaMDcz9mxXgcTRdCFLyDcnkUU07liRBmRoLxC/vj5tjEp4C0G6uSfR5+WEMmf3duWCZf6JvZNF1IsUwc5NYRv hydra-poc-builder@iog-hydra.iam.gserviceaccount.com diff --git a/mithril-infra/main.firewall.tf b/mithril-infra/main.firewall.tf index 9b61d40a9e9..c51ead45e7e 100644 --- a/mithril-infra/main.firewall.tf +++ b/mithril-infra/main.firewall.tf @@ -1,5 +1,5 @@ locals { - p2p_network_ports = var.mithril_use_p2p_network ? concat([local.mithril_aggregator_relay_mithril_listen_port], values(local.mithril_signers_relay_listen_port)) : [] + p2p_network_ports = var.mithril_use_p2p_network ? concat([local.mithril_aggregator_relay_mithril_listen_port, local.mithril_aggregator_dmq_port], values(local.mithril_signers_relay_listen_port)) : [] } resource "google_compute_firewall" "mithril-vm-firewall" { diff --git a/mithril-infra/mithril.aggregator.tf b/mithril-infra/mithril.aggregator.tf index 10d01abd189..1040ff29b58 100644 --- a/mithril-infra/mithril.aggregator.tf +++ b/mithril-infra/mithril.aggregator.tf @@ -1,5 +1,6 @@ locals { mithril_aggregator_relay_mithril_listen_port = 6060 + mithril_aggregator_dmq_port = 6161 mithril_aggregator_ancillary_signer_gcp_kms_credentials = base64decode(var.mithril_aggregator_ancillary_signer_gcp_kms_credentials) } @@ -68,6 +69,31 @@ if [ "$FOUND_CONFIGURATION" = "false" ]; then echo "No cardano node configuration found for version $CARDANO_NODE_EXACT_VERSION" exit 1 fi +EOT +, + <<-EOT +set -e +# Setup dmq node configuration +AGGREGATOR_CONFIG_DIRECTORY=/home/curry/data/${var.cardano_network}/mithril-aggregator/dmq +rm -rf $AGGREGATOR_CONFIG_DIRECTORY +mkdir -p $AGGREGATOR_CONFIG_DIRECTORY +cp -R /home/curry/docker/dmq/config/ $AGGREGATOR_CONFIG_DIRECTORY + +# Setup dmq node ipc folder (to avoid permission issues) +mkdir -p $AGGREGATOR_CONFIG_DIRECTORY/ipc + +# Setup dmq node config +cat $AGGREGATOR_CONFIG_DIRECTORY/config/config.json | jq '. + {"CardanoNetworkMagic": ${var.cardano_network_magic_map[var.cardano_network]}, "CardanoNodeSocket": "/ipc-cardano/node.socket"}' > $AGGREGATOR_CONFIG_DIRECTORY/config/config.json.new +rm -f $AGGREGATOR_CONFIG_DIRECTORY/config/config.json +mv $AGGREGATOR_CONFIG_DIRECTORY/config/config.json.new $AGGREGATOR_CONFIG_DIRECTORY/config/config.json + +# Setup dmq node topology for bootstrap peer +if [ "${var.mithril_p2p_network_bootstrap_peer}" != "" ]; then + cat $AGGREGATOR_CONFIG_DIRECTORY/config/topology.json | jq 'del(.bootstrapPeers)' | jq '.localRoots[0].accessPoints += [{ "address": "${element(split(":",var.mithril_p2p_network_bootstrap_peer),0)}", "port": ${element(split(":",var.mithril_p2p_network_bootstrap_peer),1)}, "valency": 1}]' > $AGGREGATOR_CONFIG_DIRECTORY/config/topology.json.new + rm -f $AGGREGATOR_CONFIG_DIRECTORY/config/topology.json + mv $AGGREGATOR_CONFIG_DIRECTORY/config/topology.json.new $AGGREGATOR_CONFIG_DIRECTORY/config/topology.json + exit 0 +fi EOT ] } @@ -132,7 +158,10 @@ EOT , "export LEADER_AGGREGATOR_ENDPOINT='${var.mithril_aggregator_leader_aggregator_endpoint}'", "export AGGREGATOR_RELAY_LISTEN_PORT='${local.mithril_aggregator_relay_mithril_listen_port}'", + "export AGGREGATOR_DMQ_ADDR='0.0.0.0'", + "export AGGREGATOR_DMQ_PORT='${local.mithril_aggregator_dmq_port}'", "export P2P_BOOTSTRAP_PEER='${var.mithril_p2p_network_bootstrap_peer}'", + "export DMQ_NODE_BINARY_URL='${var.mithril_p2p_dmq_node_binary_url}'", "export ENABLE_METRICS_SERVER=true", "export METRICS_SERVER_IP=0.0.0.0", "export METRICS_SERVER_PORT=9090", @@ -147,8 +176,9 @@ DOCKER_COMPOSE_FILES="-f $DOCKER_DIRECTORY/docker-compose-aggregator-base.yaml" if [ "${local.mithril_aggregator_use_authentication}" = "true" ]; then DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-aggregator-auth-override.yaml" fi -# Support for aggregator P2P network -if [ "${var.mithril_use_p2p_network}" = "true" ]; then +# Support for aggregator P2P network (without real DMQ node) +if [ "${var.mithril_use_p2p_network}" = "true" ] && [ "${var.mithril_p2p_use_real_dmq_node}" = "false" ]; then + DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-aggregator-p2p-base-override.yaml" if [ "${var.mithril_p2p_network_bootstrap_peer}" != "" ]; then @@ -161,7 +191,11 @@ if [ "${local.mithril_aggregator_is_follower}" = "true" ]; then fi # Support for DMQ protocol if [ "${var.mithril_p2p_use_dmq_protocol}" = "true" ]; then - DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-aggregator-p2p-dmq-override.yaml" + if [ "${var.mithril_p2p_use_real_dmq_node}" = "true" ]; then + DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-aggregator-p2p-dmq-real-node-override.yaml" + else + DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-aggregator-p2p-dmq-fake-node-override.yaml" + fi fi EOT , diff --git a/mithril-infra/mithril.signer.tf b/mithril-infra/mithril.signer.tf index 0905ff59840..8e419e35a86 100644 --- a/mithril-infra/mithril.signer.tf +++ b/mithril-infra/mithril.signer.tf @@ -5,6 +5,7 @@ locals { mithril_signers_relay_server_port = { for key, signer in var.mithril_signers : key => index(local.mithril_signers_index, key) + 1 + 7070 } mithril_signers_relay_cardano_port = { for key, signer in var.mithril_signers : key => index(local.mithril_signers_index, key) + 1 + 9090 } mithril_signers_block_producer_cardano_port = { for key, signer in var.mithril_signers : key => index(local.mithril_signers_index, key) + 1 + 10000 } + mithril_signers_dmq_port = { for key, signer in var.mithril_signers : key => index(local.mithril_signers_index, key) + 1 + 11000 } } resource "null_resource" "mithril_signer" { @@ -93,6 +94,36 @@ for SIGNER_TYPE in $SIGNER_TYPES; do fi done +EOT +, + <<-EOT +set -e +# Setup dmq node configuration +SIGNER_TYPE_CONFIG_DIRECTORY=/home/curry/data/${var.cardano_network}/mithril-signer-${each.key}/dmq +rm -rf $SIGNER_TYPE_CONFIG_DIRECTORY +mkdir -p $SIGNER_TYPE_CONFIG_DIRECTORY +cp -R /home/curry/docker/dmq/config/ $SIGNER_TYPE_CONFIG_DIRECTORY + +# Setup dmq node ipc folder (to avoid permission issues) +mkdir -p $SIGNER_TYPE_CONFIG_DIRECTORY/ipc + +# Setup dmq node config +cat $SIGNER_TYPE_CONFIG_DIRECTORY/config/config.json | jq '. + {"CardanoNetworkMagic": ${var.cardano_network_magic_map[var.cardano_network]}, "CardanoNodeSocket": "/ipc-cardano/node.socket"}' > $SIGNER_TYPE_CONFIG_DIRECTORY/config/config.json.new +rm -f $SIGNER_TYPE_CONFIG_DIRECTORY/config/config.json +mv $SIGNER_TYPE_CONFIG_DIRECTORY/config/config.json.new $SIGNER_TYPE_CONFIG_DIRECTORY/config/config.json + +# Setup dmq node topology for aggregator +cat $SIGNER_TYPE_CONFIG_DIRECTORY/config/topology.json | jq 'del(.bootstrapPeers)' | jq '.localRoots[0].accessPoints += [{ "address": "${google_compute_address.mithril-external-address.address}", "port": ${local.mithril_aggregator_dmq_port}, "valency": 1}]' > $SIGNER_TYPE_CONFIG_DIRECTORY/config/topology.json.new +rm -f $SIGNER_TYPE_CONFIG_DIRECTORY/config/topology.json +mv $SIGNER_TYPE_CONFIG_DIRECTORY/config/topology.json.new $SIGNER_TYPE_CONFIG_DIRECTORY/config/topology.json + +# Setup dmq node topology for bootstrap peer +if [ "${var.mithril_p2p_network_bootstrap_peer}" != "" ]; then + cat $SIGNER_TYPE_CONFIG_DIRECTORY/config/topology.json | jq 'del(.bootstrapPeers)' | jq '.localRoots[0].accessPoints += [{ "address": "${element(split(":",var.mithril_p2p_network_bootstrap_peer),0)}", "port": ${element(split(":",var.mithril_p2p_network_bootstrap_peer),1)}, "valency": 1}]' > $SIGNER_TYPE_CONFIG_DIRECTORY/config/topology.json.new + rm -f $SIGNER_TYPE_CONFIG_DIRECTORY/config/topology.json + mv $SIGNER_TYPE_CONFIG_DIRECTORY/config/topology.json.new $SIGNER_TYPE_CONFIG_DIRECTORY/config/topology.json + exit 0 +fi EOT ] } @@ -113,6 +144,8 @@ EOT "export SIGNER_CARDANO_RELAY_PORT=${local.mithril_signers_relay_cardano_port[each.key]}", "export SIGNER_CARDANO_BLOCK_PRODUCER_ADDR=0.0.0.0", "export SIGNER_CARDANO_BLOCK_PRODUCER_PORT=${local.mithril_signers_block_producer_cardano_port[each.key]}", + "export SIGNER_DMQ_ADDR='0.0.0.0'", + "export SIGNER_DMQ_PORT=${local.mithril_signers_dmq_port[each.key]}", "export ERA_READER_ADAPTER_TYPE='${var.mithril_era_reader_adapter_type}'", <<-EOT ERA_READER_ADAPTER_PARAMS=$(jq -nc --arg address $(wget -q -O - ${var.mithril_era_reader_address_url}) --arg verification_key $(wget -q -O - ${var.mithril_era_reader_verification_key_url}) '{"address": $address, "verification_key": $verification_key}') @@ -134,6 +167,7 @@ fi EOT , "export P2P_BOOTSTRAP_PEER='${var.mithril_p2p_network_bootstrap_peer}'", + "export DMQ_NODE_BINARY_URL='${var.mithril_p2p_dmq_node_binary_url}'", "export ENABLE_METRICS_SERVER=true", "export METRICS_SERVER_IP=0.0.0.0", "export METRICS_SERVER_PORT=9090", @@ -176,15 +210,24 @@ if [ "${each.value.type}" = "unverified-cardano-shared-norelay" ]; then fi # Support for signer P2P network if [ "${var.mithril_use_p2p_network}" = "true" ]; then - DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-signer-p2p-base-override.yaml" - - if [ "${var.mithril_p2p_network_bootstrap_peer}" != "" ]; then - DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-signer-p2p-bootstrap-override.yaml" + if [ "${var.mithril_p2p_use_real_dmq_node}" = "true" ]; then + # TODO: Add support for DMQ bootstrap + echo "DMQ node support not yet implemented in this script" + else + DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-signer-p2p-base-no-dmq-override.yaml" + + if [ "${var.mithril_p2p_network_bootstrap_peer}" != "" ]; then + DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-signer-p2p-bootstrap-no-dmq-override.yaml" + fi fi fi # Support for DMQ protocol if [ "${var.mithril_p2p_use_dmq_protocol}" = "true" ]; then - DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-signer-p2p-dmq-override.yaml" + if [ "${var.mithril_p2p_use_real_dmq_node}" = "true" ]; then + DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-signer-p2p-dmq-real-node-override.yaml" + else + DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-signer-p2p-dmq-fake-node-override.yaml" + fi fi EOT , diff --git a/mithril-infra/variables.tf b/mithril-infra/variables.tf index 0ffead79db1..463f7fd6f47 100644 --- a/mithril-infra/variables.tf +++ b/mithril-infra/variables.tf @@ -200,6 +200,19 @@ variable "mithril_p2p_use_dmq_protocol" { default = false } +variable "mithril_p2p_use_real_dmq_node" { + type = bool + description = "Use the real Decentralized Message Queue (DMQ) node (experimental, for test only)" + default = false +} + + +variable "mithril_p2p_dmq_node_binary_url" { + type = string + description = "The url to download the DMQ node binary (experimental, for test only)" + default = "" +} + variable "mithril_p2p_network_bootstrap_peer" { type = string description = "The dial to address of a bootstrap peer of the P2P network layer. Useful when setting-up a follower aggregator and signers in a different VM. (experimental, for test only)" diff --git a/mithril-test-lab/mithril-devnet/bin/dmq-node b/mithril-test-lab/mithril-devnet/bin/dmq-node new file mode 100755 index 00000000000..0e3cd94498c Binary files /dev/null and b/mithril-test-lab/mithril-devnet/bin/dmq-node differ diff --git a/mithril-test-lab/mithril-devnet/mkfiles/mkfiles-init.sh b/mithril-test-lab/mithril-devnet/mkfiles/mkfiles-init.sh index 09acb19a59b..90d88d3b2d9 100644 --- a/mithril-test-lab/mithril-devnet/mkfiles/mkfiles-init.sh +++ b/mithril-test-lab/mithril-devnet/mkfiles/mkfiles-init.sh @@ -79,6 +79,10 @@ if [[ "$SKIP_CARDANO_BIN_DOWNLOAD" != "true" ]]; then rm -f cardano-bin.tar.gz fi +# TODO: remove this temporary implementation to copy DMQ node binary once the DMQ is part of the Cardano node bundle +cp ./bin/dmq-node ${ARTIFACTS_DIR}/bin/ +chmod u+x ${ARTIFACTS_DIR}/bin/dmq-node + # Switch to artifacts directory pushd "${ARTIFACTS_DIR}" > /dev/null || exit