feat: DMQ node infrastructure

.github/workflows/ci.yml

@@ -389,7 +389,10 @@ jobs:
         cardano_node_version: ["10.4.1", "10.5.1"]
         hard_fork_latest_era_at_epoch: [0]
         run_id: ["#1", "#2", "#3"]
-        extra_args: ["--aggregate-signature-type=Concatenation"]
+        extra_args:
+          [
+            "--aggregate-signature-type=Concatenation --use-dmq --dmq-node-flavor=haskell",
+          ]
 
         include:
           # Include a test for partial decentralization with leader/follower signer registration and P2P signature registration with fake DMQ node
@@ -400,7 +403,15 @@ jobs:
             hard_fork_latest_era_at_epoch: 0
             run_id: "#1"
             extra_args: "--number-of-aggregators=2 --use-relays --relay-signer-registration-mode=passthrough --relay-signature-registration-mode=p2p --aggregate-signature-type=Concatenation --use-dmq --dmq-node-flavor=fake"
-          # Include a test for partial decentralization with leader/follower signer registration and P2P signature registration
+          # Include a test for partial decentralization with leader/follower signer registration and P2P signature registration with fake DMQ node
+          - mode: "leader-follower"
+            era: ${{ fromJSON(needs.build-ubuntu.outputs.eras)[0] }}
+            next_era: [""]
+            cardano_node_version: "10.5.1"
+            hard_fork_latest_era_at_epoch: 0
+            run_id: "#1"
+            extra_args: "--number-of-aggregators=2 --use-relays --relay-signer-registration-mode=passthrough --relay-signature-registration-mode=p2p --aggregate-signature-type=Concatenation --use-dmq --dmq-node-flavor=fake"
+          # Include a test for partial decentralization with leader/follower signer registration and P2P signature registration with Mithril relay
           - mode: "leader-follower"
             era: ${{ fromJSON(needs.build-ubuntu.outputs.eras)[0] }}
             next_era: [""]

.github/workflows/test-deploy-network.yml

@@ -89,7 +89,7 @@ jobs:
               {
                 "1" = {
                   type    = "unverified-cardano-passive-norelay",
-                  pool_id = "pool1y0uxkqyplyx6ld25e976t0s35va3ysqcscatwvy2sd2cwcareq7",
+                  pool_id = "pool13zafxlpfgymf474uv52qt557z5k5frn9p83yr55zp267wj5mpu4",
                 },
               }
             mithril_leader_aggregator_endpoint: https://aggregator.dev-preview.api.mithril.network/aggregator

docs/website/root/manual/operate/run-signer-node.md

@@ -99,10 +99,12 @@ Note that this guide works on a Linux machine only.
 :::
 
 - To operate a **Cardano node** as a **stake pool**, you need:
+
   - The pool's `operational certificate`
   - The pool's `KES secret key`
 
 - To access the file system of the **Cardano block producer** node for **production** deployment (or of the **Cardano relay** node for **naive** deployment), you will need the following permissions:
+
   - Read rights on the `Database` folder (specified by the `--database-path` setting of the **Cardano node**)
   - Read and write rights on the `Inter Process Communication` file (typically defined by the `CARDANO_NODE_SOCKET_PATH` environment variable used to launch the **Cardano node**)
 
@@ -546,6 +548,7 @@ sudo systemctl restart mithril-signer
 :::info
 
 - If you have already installed `Squid` via `apt` package manager, we recommend that you delete it before manually building it from source by running the following commands:
+
   - `sudo systemctl stop squid`
   - `sudo apt remove squid`
   - `sudo apt autoremove`.
@@ -889,6 +892,53 @@ sudo service netfilter-persistent save
 
 :::
 
+## Setup the DMQ node (unstable)
+
+:::caution
+
+The DMQ node setup is currently **unstable** and not suitable for production use.
+
+:::
+
+The DMQ node supports the implementation of a **Decentralized Message Queue** (DMQ) for Mithril. The DMQ protocol allows Mithril signers to exchange signatures in a decentralized manner, enhancing the robustness and scalability of the Mithril networks. Once stabilized and deployed on a majority of SPOs on a Mithril network, the DMQ protocol will allow multiple aggregators to operate simultaneously, improving the overall availability of the Mithril protocol. The DMQ protocol is fully described in the [CIP-0137](https://cips.cardano.org/cip/CIP-0137).
+
+### Download the pre-built binary
+
+:::tip
+
+You can use these parameters for the **pre-release-preview** network:
+
+- **DMQ_RELEASE_URL**: `https://github.com/input-output-hk/mithril/raw/refs/heads/jpraynaud/dmq-node-binary/mithril-test-lab/mithril-devnet/bin/dmq-node-0.2.0.0-53bf9652787dc768abd86cf3844f1206f0fd7d8c`
+
+This URL may change in the future; please refer to this page for the latest released version of the DMQ node binary.
+
+:::
+
+To download the latest released version of the DMQ node binary, run the following command:
+
+```bash
+curl --fail -sL -o dmq-node **DMQ_RELEASE_URL**
+```
+
+### Installing the service
+
+#### Make the binary executable
+
+To make the binary executable, run:
+
+```bash
+chmod +x dmq-node
+```
+
+#### Move the executable
+
+To move the executable to /opt/mithril, run:
+
+```bash
+sudo mkdir -p /opt/mithril
+sudo mv dmq-node /opt/mithril
+```
+
 ## Verify the Mithril signer deployment
 
 :::tip

mithril-common/Cargo.toml

@@ -19,7 +19,7 @@ rustdoc-args = ["--cfg", "docsrs"]
 ignored = ["serde_bytes"]
 
 [features]
-default = ["rug-backend"]
+default = ["rug-backend", "allow_skip_signer_certification"]
 
 # Enables `rug-backend` features for `mithril-stm` dependency
 rug-backend = ["mithril-stm/rug-backend"]

mithril-infra/assets/docker/Dockerfile.dmq

@@ -0,0 +1,11 @@
+FROM alpine:latest
+ARG DMQ_NODE_BINARY_URL
+ARG MITHRIL_IMAGE_ID # Only for triggering rebuilds when mithril image changes
+
+RUN apk add --no-cache wget
+
+RUN wget -O /bin/dmq-node $DMQ_NODE_BINARY_URL 
+RUN chmod +x /bin/dmq-node
+RUN /bin/dmq-node --version
+
+ENTRYPOINT ["/bin/dmq-node"]

mithril-infra/assets/docker/dmq/config/config.json

@@ -0,0 +1,17 @@
+{
+  "LocalMsgSubmissionTracer": true,
+  "LocalMsgNotificationTracer": true,
+  "ConnectionManagerTracer": true,
+  "DiffusionTracer": false,
+  "InboundGovernorTracer": false,
+  "LocalInboundGovernorTracer": false,
+  "PeerSelectionTracer": false,
+  "PeerSelectionCounters": false,
+  "PeerSharing": false,
+  "SigSubmissionLogicTracer": false,
+  "SigSubmissionClientTracer": false,
+  "SigSubmissionServerTracer": false,
+  "MuxTracer": false,
+  "ChannelTracer": false,
+  "DebugPeerSelectionTracer": false
+}

mithril-infra/assets/docker/dmq/config/topology.json

@@ -0,0 +1,18 @@
+{
+  "bootstrapPeers": [],
+  "localRoots": [
+    {
+      "accessPoints": [],
+      "advertise": false,
+      "trustable": false,
+      "valency": 2
+    }
+  ],
+  "peerSnapshotFile": null,
+  "publicRoots": [
+    {
+      "accessPoints": [],
+      "advertise": false
+    }
+  ]
+}

mithril-infra/assets/docker/docker-compose-aggregator-p2p-dmq-real-node-override.yaml

@@ -0,0 +1,52 @@
+services:
+  mithril-aggregator:
+    volumes:
+      - ../data/${NETWORK}/mithril-aggregator/dmq/ipc:/ipc-dmq
+    environment:
+      - DMQ_NODE_SOCKET_PATH=/ipc-dmq/dmq.socket
+      - NETWORK=${NETWORK}
+      - NETWORK_MAGIC=${NETWORK_MAGIC}
+
+  dmq-node:
+    # TODO: gather logs and send to loki
+    image: dmq-node-${MITHRIL_IMAGE_ID}
+    container_name: dmq-node-aggregator
+    restart: always
+    user: ${CURRENT_UID}
+    profiles:
+      - dmq
+      - all
+    build:
+      context: .
+      dockerfile: Dockerfile.dmq
+      args:
+        DMQ_NODE_BINARY_URL: ${DMQ_NODE_BINARY_URL}
+        MITHRIL_IMAGE_ID: ${MITHRIL_IMAGE_ID}
+    volumes:
+      - ../data/${NETWORK}/mithril-aggregator/dmq/config:/config
+      - ../data/${NETWORK}/mithril-aggregator/dmq/ipc:/ipc
+      - ../data/${NETWORK}/mithril-aggregator/cardano/ipc:/ipc-cardano
+    ports:
+      - "${AGGREGATOR_DMQ_PORT}:${AGGREGATOR_DMQ_PORT}"
+    logging:
+      driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
+    command:
+      [
+        "--configuration-file",
+        "/config/config.json",
+        "--topology-file",
+        "/config/topology.json",
+        "--local-socket",
+        "/ipc/dmq.socket",
+        "--host-addr",
+        "${AGGREGATOR_DMQ_ADDR}",
+        "--port",
+        "${AGGREGATOR_DMQ_PORT}",
+      ]
+
+networks:
+  default:
+    name: mithril_network
+    external: true

mithril-infra/assets/docker/docker-compose-reverse-proxy.yaml

@@ -1,6 +1,6 @@
 services:
   reverse-proxy:
-    image: traefik:3.3
+    image: traefik:3.6
     container_name: reverse-proxy
     restart: always
     user: "${CURRENT_UID}:${DOCKER_GID}"

mithril-infra/assets/docker/docker-compose-signer-p2p-dmq-real-node-override.yaml

@@ -0,0 +1,50 @@
+services:
+  mithril-signer:
+    volumes:
+      - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/dmq/ipc:/ipc-dmq
+    environment:
+      - DMQ_NODE_SOCKET_PATH=/ipc-dmq/dmq.socket
+      - NETWORK_MAGIC=${NETWORK_MAGIC}
+      - SIGNATURE_PUBLISHER_SKIP_DELAYER=true # TODO: remove
+  dmq-node:
+    image: dmq-node-${MITHRIL_IMAGE_ID}
+    container_name: dmq-node-signer-${SIGNER_ID}
+    restart: always
+    user: ${CURRENT_UID}
+    profiles:
+      - dmq
+      - all
+    build:
+      context: .
+      dockerfile: Dockerfile.dmq
+      args:
+        DMQ_NODE_BINARY_URL: ${DMQ_NODE_BINARY_URL}
+        MITHRIL_IMAGE_ID: ${MITHRIL_IMAGE_ID}
+    volumes:
+      - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/dmq/config:/config
+      - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/dmq/ipc:/ipc
+      - ../data/${NETWORK}/mithril-signer-${SIGNER_ID}/cardano/ipc:/ipc-cardano
+    ports:
+      - "${SIGNER_DMQ_PORT}:${SIGNER_DMQ_PORT}"
+    logging:
+      driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
+    command:
+      [
+        "--configuration-file",
+        "/config/config.json",
+        "--topology-file",
+        "/config/topology.json",
+        "--local-socket",
+        "/ipc/dmq.socket",
+        "--host-addr",
+        "${SIGNER_DMQ_ADDR}",
+        "--port",
+        "${SIGNER_DMQ_PORT}",
+      ]
+
+networks:
+  default:
+    name: mithril_network
+    external: true

mithril-infra/assets/ssh_keys-testing

@@ -2,3 +2,4 @@ curry:ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3bO4CpX+S2y1X9Q2v7L9sPV5ZiGgo3B4rv4k
 root:ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF3bO4CpX+S2y1X9Q2v7L9sPV5ZiGgo3B4rv4k3pgmko jpraynaud
 curry:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDkI86FAS+Bdce+gr0BpfyijTmrX+I/smi8xOsPVSrxFwQbo5HYwbuaLzQ2RfxEvb+rwaVyeIMLNGe5jWTYc/+tYTV40aQ8PkrUTcl7olazkelHbkgxq1ScN/GQvu7xDg+vjiH5vO7AZOn/NWq9CKeSB8rb/kZvcX1uWP+NygKAthYo33AFw9XoEhO/GUVSgG1M8ZRg5oSdJYZNvUiEGd/phe5OlwWH/4IcPOuyR4M9FAMAv9WSBzcAfQNO0SHJjBMFUkGAfw1aTqd0GPAfrRBLkJ2XyBRBkDKp72WVmPIdwDihoFIo1rOYvw0qQby//vbKpbTrL2cItNbqTwO2UY3FVkZ5MhJp9QaUUr/Ocie+KbE3jFeiVirIqxPksR+O19xAevKz2XL8RlICuZSBDovhjqBzvow36OdCm5mdCLX0psOSeIAS4I0iZaSooo+z1q4aXM+wYMgJxUF2IHr7TdGbHThtsDysPVuVlCai2+c/XC7IwIK1wrm+y5W0wPu6Vu0= alenar
 curry:ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBce8iKP5p4FI1+vLvr1R7NtIcwwyIEh4bRC1fyI+6oT turmelclem
+curry:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNGKX8fIDYbnc0eBY+IjD6wBMEKHjCNeWKeCAmt+KbGlF/r2Z4POLIZb4eUNs/P9BQn1d85V8dR64TpM05Af7n9bpQFN5+MOgFD7/o78gGJXKHrgnXemuMh7Q3LMqjgDIb+WixDDYDq1c68u03QM6G/7qYCotjoE/pQI0o7143RSnkQQGxFSzZ9WTgrgvKJK+6ENpC6ItGLm4rZAzdyez/3sOkT8JO+3Da8xFJVf7Ladf91lpwONAIvGCROG7bZl0NaMDcz9mxXgcTRdCFLyDcnkUU07liRBmRoLxC/vj5tjEp4C0G6uSfR5+WEMmf3duWCZf6JvZNF1IsUwc5NYRv hydra-poc-builder@iog-hydra.iam.gserviceaccount.com

mithril-infra/main.firewall.tf

@@ -1,5 +1,5 @@
 locals {
-  p2p_network_ports = var.mithril_use_p2p_network ? concat([local.mithril_aggregator_relay_mithril_listen_port], values(local.mithril_signers_relay_listen_port)) : []
+  p2p_network_ports = var.mithril_use_p2p_network ? concat([local.mithril_aggregator_relay_mithril_listen_port, local.mithril_aggregator_dmq_port], values(local.mithril_signers_relay_listen_port)) : []
 }
 
 resource "google_compute_firewall" "mithril-vm-firewall" {

mithril-infra/mithril.aggregator.tf

@@ -1,5 +1,6 @@
 locals {
   mithril_aggregator_relay_mithril_listen_port            = 6060
+  mithril_aggregator_dmq_port                             = 6161
   mithril_aggregator_ancillary_signer_gcp_kms_credentials = base64decode(var.mithril_aggregator_ancillary_signer_gcp_kms_credentials)
 }
 
@@ -68,6 +69,31 @@ if [ "$FOUND_CONFIGURATION" = "false" ]; then
   echo "No cardano node configuration found for version $CARDANO_NODE_EXACT_VERSION"
   exit 1
 fi
+EOT
+,
+      <<-EOT
+set -e
+# Setup dmq node configuration
+AGGREGATOR_CONFIG_DIRECTORY=/home/curry/data/${var.cardano_network}/mithril-aggregator/dmq
+rm -rf $AGGREGATOR_CONFIG_DIRECTORY
+mkdir -p $AGGREGATOR_CONFIG_DIRECTORY
+cp -R /home/curry/docker/dmq/config/ $AGGREGATOR_CONFIG_DIRECTORY
+
+# Setup dmq node ipc folder (to avoid permission issues)
+mkdir -p $AGGREGATOR_CONFIG_DIRECTORY/ipc
+
+# Setup dmq node config
+cat $AGGREGATOR_CONFIG_DIRECTORY/config/config.json | jq '. + {"CardanoNetworkMagic": ${var.cardano_network_magic_map[var.cardano_network]}, "CardanoNodeSocket": "/ipc-cardano/node.socket"}' > $AGGREGATOR_CONFIG_DIRECTORY/config/config.json.new
+rm -f $AGGREGATOR_CONFIG_DIRECTORY/config/config.json
+mv $AGGREGATOR_CONFIG_DIRECTORY/config/config.json.new $AGGREGATOR_CONFIG_DIRECTORY/config/config.json
+
+# Setup dmq node topology for bootstrap peer
+if [ "${var.mithril_p2p_network_bootstrap_peer}" != "" ]; then
+  cat $AGGREGATOR_CONFIG_DIRECTORY/config/topology.json | jq 'del(.bootstrapPeers)' | jq '.localRoots[0].accessPoints += [{ "address": "${element(split(":",var.mithril_p2p_network_bootstrap_peer),0)}", "port": ${element(split(":",var.mithril_p2p_network_bootstrap_peer),1)}, "valency": 1}]' > $AGGREGATOR_CONFIG_DIRECTORY/config/topology.json.new
+  rm -f $AGGREGATOR_CONFIG_DIRECTORY/config/topology.json
+  mv $AGGREGATOR_CONFIG_DIRECTORY/config/topology.json.new $AGGREGATOR_CONFIG_DIRECTORY/config/topology.json
+  exit 0
+fi
 EOT
     ]
   }
@@ -132,7 +158,10 @@ EOT
       ,
       "export LEADER_AGGREGATOR_ENDPOINT='${var.mithril_aggregator_leader_aggregator_endpoint}'",
       "export AGGREGATOR_RELAY_LISTEN_PORT='${local.mithril_aggregator_relay_mithril_listen_port}'",
+      "export AGGREGATOR_DMQ_ADDR='0.0.0.0'",
+      "export AGGREGATOR_DMQ_PORT='${local.mithril_aggregator_dmq_port}'",
       "export P2P_BOOTSTRAP_PEER='${var.mithril_p2p_network_bootstrap_peer}'",
+      "export DMQ_NODE_BINARY_URL='${var.mithril_p2p_dmq_node_binary_url}'",
       "export ENABLE_METRICS_SERVER=true",
       "export METRICS_SERVER_IP=0.0.0.0",
       "export METRICS_SERVER_PORT=9090",
@@ -147,8 +176,9 @@ DOCKER_COMPOSE_FILES="-f $DOCKER_DIRECTORY/docker-compose-aggregator-base.yaml"
 if [ "${local.mithril_aggregator_use_authentication}" = "true" ]; then
   DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-aggregator-auth-override.yaml"
 fi
-# Support for aggregator P2P network
-if [ "${var.mithril_use_p2p_network}" = "true" ]; then
+# Support for aggregator P2P network (without real DMQ node)
+if [ "${var.mithril_use_p2p_network}" = "true" ] && [ "${var.mithril_p2p_use_real_dmq_node}" = "false" ]; then
+
   DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-aggregator-p2p-base-override.yaml"
 
   if [ "${var.mithril_p2p_network_bootstrap_peer}" != "" ]; then
@@ -161,7 +191,11 @@ if [ "${local.mithril_aggregator_is_follower}" = "true" ]; then
 fi
 # Support for DMQ protocol
 if [ "${var.mithril_p2p_use_dmq_protocol}" = "true" ]; then
- DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-aggregator-p2p-dmq-override.yaml"
+  if [ "${var.mithril_p2p_use_real_dmq_node}" = "true" ]; then
+    DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-aggregator-p2p-dmq-real-node-override.yaml"
+  else
+    DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f $DOCKER_DIRECTORY/docker-compose-aggregator-p2p-dmq-fake-node-override.yaml"
+  fi
 fi
 EOT
       ,