Detailed statuses for runtimes

Gemfile

@@ -79,7 +79,7 @@ gem 'good_job', '~> 4.0'
 gem 'rotp'
 
 gem 'grpc', '~> 1.67'
-gem 'tucana', '0.0.47'
+gem 'tucana', '0.0.0', path: '../tucana/build/ruby'
 
 gem 'code0-identities', '~> 0.0.2'
 

Gemfile.lock

@@ -1,3 +1,9 @@
+PATH
+  remote: ../tucana/build/ruby
+  specs:
+    tucana (0.0.0)
+      grpc (~> 1.64)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -376,8 +382,6 @@ GEM
     thor (1.4.0)
     timeout (0.4.4)
     tsort (0.2.0)
-    tucana (0.0.47)
-      grpc (~> 1.64)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (2.6.0)
@@ -430,7 +434,7 @@ DEPENDENCIES
   simplecov (~> 0.22.0)
   simplecov-cobertura (~> 3.0)
   test-prof (~> 1.0)
-  tucana (= 0.0.47)
+  tucana (= 0.0.0)!
   tzinfo-data
 
 RUBY VERSION

app/graphql/types/runtime_status_configuration_endpoint_type.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Types
+  class RuntimeStatusConfigurationEndpointType < Types::BaseObject
+    description 'Detailed information about a runtime status'
+
+    field :endpoint, String, null: false, description: 'The endpoint URL of the runtime'
+
+    id_field ::RuntimeStatusConfiguration
+    timestamps
+  end
+end

app/graphql/types/runtime_status_configuration_type.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Types
+  class RuntimeStatusConfigurationType < Types::BaseUnion
+    description 'Detailed configuration about a runtime status, either: endpoint, ...'
+
+    possible_types Types::RuntimeStatusConfigurationEndpointType
+
+    def self.resolve_type(object, _context)
+      return Types::RuntimeStatusConfigurationEndpointType if object.endpoint.present?
+
+      raise "Unknown RuntimeStatusInformation type: #{object.class.name}"
+    end
+  end
+end

app/graphql/types/runtime_status_enum.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Types
+  class RuntimeStatusEnum < BaseEnum
+    description 'Represent all available aquila statuses'
+
+    value :CONNECTED, 'No problem with the connection to aquila', value: :connected
+    value :DISCONNECTED, 'The runtime is disconnected, cause unknown', value: :disconnected
+  end
+end

app/graphql/types/runtime_status_status_enum.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Types
+  class RuntimeStatusStatusEnum < Types::BaseEnum
+    description 'The enum status of the detailed status'
+
+    value :NOT_RESPONDING, 'The runtime is not responding to heartbeats', value: 'NOT_RESPONDING'
+    value :NOT_READY, 'The runtime is not ready to compute stuff', value: 'NOT_READY'
+    value :RUNNING, 'The runtime is running and healthy', value: 'RUNNING'
+    value :STOPPED, 'The runtime has been stopped', value: 'STOPPED'
+  end
+end

app/graphql/types/runtime_status_type.rb

@@ -1,10 +1,25 @@
 # frozen_string_literal: true
 
 module Types
-  class RuntimeStatusType < BaseEnum
-    description 'Represent all available types of statuses of a runtime'
+  class RuntimeStatusType < Types::BaseObject
+    description 'A runtime status information entry'
 
-    value :CONNECTED, 'No problem with connection, everything works as expected', value: 'connected'
-    value :DISCONNECTED, 'The runtime is disconnected, cause unknown', value: 'disconnected'
+    field :configurations, Types::RuntimeStatusConfigurationType.connection_type,
+          null: false,
+          description: 'The detailed configuration entries for this runtime status (only for adapters)',
+          method: :runtime_status_configurations
+    field :features, [String], null: false, description: 'The set of features supported by the runtime'
+    field :identifier, String, null: false, description: 'The unique identifier for this runtime status'
+    field :last_heartbeat, Types::TimeType, null: true,
+                                            description: 'The timestamp of the last heartbeat received from the runtime'
+    field :status, Types::RuntimeStatusStatusEnum,
+          null: false,
+          description: 'The current status of the runtime (e.g. running, stopped)'
+    field :type, Types::RuntimeStatusTypeEnum,
+          null: false,
+          description: 'The type of runtime status information (e.g. adapter, execution)', method: :status_type
+
+    id_field RuntimeStatus
+    timestamps
   end
 end

app/graphql/types/runtime_status_type_enum.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Types
+  class RuntimeStatusTypeEnum < Types::BaseEnum
+    description 'The type of runtime status'
+
+    value :ADAPTER, 'Indicates that the runtime status is related to an adapter.', value: 'adapter'
+    value :EXECUTION, 'Indicates that the runtime status is related to an execution.', value: 'execution'
+  end
+end

app/graphql/types/runtime_type.rb

@@ -16,10 +16,16 @@ class RuntimeType < Types::BaseObject
     field :runtime_function_definitions, Types::RuntimeFunctionDefinitionType.connection_type,
           null: false,
           description: 'Functions of the runtime'
-    field :status, Types::RuntimeStatusType, null: false, description: 'The status of the runtime'
 
     field :token, String, null: true, description: 'Token belonging to the runtime, only present on creation'
 
+    field :status, Types::RuntimeStatusStatusEnum,
+          null: false,
+          description: 'Wheater the last heartbeat was recent enough to consider the runtime as connected'
+    field :statuses, Types::RuntimeStatusType.connection_type, null: false,
+                                                               description: 'Statuses of the runtime',
+                                                               method: :runtime_statuses
+
     expose_abilities %i[
       delete_runtime
       update_runtime
@@ -29,6 +35,17 @@ class RuntimeType < Types::BaseObject
     id_field Runtime
     timestamps
 
+    # If the last heartbeat was within the last 10 minutes, consider the runtime as 'running'
+    def status
+      last_heartbeat = object.last_heartbeat
+
+      if last_heartbeat && last_heartbeat >= 10.minutes.ago
+        :connected
+      else
+        :disconnected
+      end
+    end
+
     def token
       object.token if object.token_previously_changed?
     end

app/grpc/flow_handler.rb

@@ -8,6 +8,9 @@ class FlowHandler < Tucana::Sagittarius::FlowService::Service
   grpc_stream :update
 
   def self.update_runtime(runtime)
+    runtime.last_heartbeat = Time.zone.now
+    runtime.save!
+
     flows = []
     runtime.project_assignments.compatible.each do |assignment|
       assignment.namespace_project.flows.each do |flow|
@@ -25,22 +28,12 @@ def self.update_runtime(runtime)
     )
   end
 
-  def self.update_started(runtime_id)
-    runtime = Runtime.find(runtime_id)
-    runtime.connected!
-    runtime.save
-
+  def self.update_started(_runtime_id)
     logger.info(message: 'Runtime connected', runtime_id: runtime.id)
 
     update_runtime(runtime)
   end
 
-  def self.update_died(runtime_id)
-    runtime = Runtime.find(runtime_id)
-    runtime.disconnected!
-    runtime.save
-  end
-
   def self.encoders = { update: ->(grpc_object) { Tucana::Sagittarius::FlowResponse.encode(grpc_object) } }
 
   def self.decoders = { update: ->(string) { Tucana::Sagittarius::FlowResponse.decode(string) } }

app/grpc/runtime_status_handler.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class RuntimeStatusHandler < Tucana::Sagittarius::RuntimeStatusService::Service
+  include Code0::ZeroTrack::Loggable
+  include GrpcHandler
+
+  def update(request, _call)
+    current_runtime = Runtime.find(Code0::ZeroTrack::Context.current[:runtime][:id])
+
+    response = Runtimes::Grpc::RuntimeStatusUpdateService.new(
+      runtime: current_runtime,
+      status_info: request.send(request.status)
+    ).execute
+
+    logger.debug("RuntimeFunctionHandler#update response: #{response.inspect}")
+
+    Tucana::Sagittarius::RuntimeStatusUpdateResponse.new(success: response.success?)
+  end
+end

app/models/runtime.rb

@@ -3,21 +3,11 @@
 class Runtime < ApplicationRecord
   include TokenAttr
 
-  STATUS_TYPES = {
-    disconnected: 0,
-    connected: 1,
-  }.with_indifferent_access
-
   belongs_to :namespace, optional: true
 
   token_attr :token, prefix: 's_rt_', length: 48
 
-  enum :status, STATUS_TYPES, default: :disconnected
-
-  validates :status, presence: true,
-                     inclusion: {
-                       in: STATUS_TYPES.keys.map(&:to_s),
-                     }
+  has_many :runtime_statuses, inverse_of: :runtime
 
   has_many :project_assignments, class_name: 'NamespaceProjectRuntimeAssignment', inverse_of: :runtime
   has_many :projects, class_name: 'NamespaceProject', through: :project_assignments, source: :namespace_project,

app/models/runtime_status.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class RuntimeStatus < ApplicationRecord
+  belongs_to :runtime, inverse_of: :runtime_statuses
+  has_many :runtime_status_configurations, inverse_of: :runtime_status
+
+  STATUS_TYPES = {
+    not_responding: 0,
+    not_ready: 1,
+    running: 2,
+    stopped: 3,
+  }.with_indifferent_access
+
+  enum :status, STATUS_TYPES, default: :stopped
+
+  STATUS_TYPE_TYPES = {
+    adapter: 0,
+    execution: 1,
+  }.with_indifferent_access
+
+  enum :status_type, STATUS_TYPE_TYPES
+
+  validates :identifier, presence: true,
+                         allow_blank: false,
+                         uniqueness: { case_sensitive: false, scope: :runtime_id }
+
+  validate :runtime_status_informations_only_for_adapter
+
+  private
+
+  def runtime_status_informations_only_for_adapter
+    return if adapter?
+    return if runtime_status_configurations.empty?
+
+    errors.add(:runtime_status_informations, :only_allowed_for_adapters)
+  end
+end

app/models/runtime_status_configuration.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class RuntimeStatusConfiguration < ApplicationRecord
+  belongs_to :runtime_status, inverse_of: :runtime_status_configurations
+
+  validates :endpoint, presence: true,
+                       allow_blank: false
+end

app/services/error_code.rb

@@ -87,6 +87,8 @@ def self.error_codes
       function_value_not_found: { description: 'The id for the function value node does not exist' },
       invalid_node_parameter: { description: 'The node parameter is invalid' },
       invalid_node_function: { description: 'The node function is invalid' },
+      invalid_runtime_status: { description: 'The runtime status is invalid because of active model errors' },
+      invalid_runtime_status_configuration: { description: 'The runtime status configuration is invalid because of active model errors' },
 
       primary_level_not_found: { description: '', deprecation_reason: 'Outdated concept' },
       secondary_level_not_found: { description: '', deprecation_reason: 'Outdated concept' },

app/services/runtimes/grpc/runtime_status_update_service.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Runtimes
+  module Grpc
+    class RuntimeStatusUpdateService
+      include Sagittarius::Database::Transactional
+      include Code0::ZeroTrack::Loggable
+
+      attr_reader :runtime, :status_info
+
+      def initialize(runtime:, status_info:)
+        @runtime = runtime
+        @status_info = status_info
+      end
+
+      def execute
+        transactional do |t|
+          runtime.last_heartbeat = Time.zone.now
+
+          unless runtime.save
+            t.rollback_and_return ServiceResponse.error(
+              message: 'Failed to update runtime heartbeat',
+              error_code: :invalid_runtime,
+              details: runtime.errors
+            )
+          end
+
+          db_status = RuntimeStatus.find_or_initialize_by(runtime: runtime,
+                                                          identifier: status_info.identifier)
+
+          db_status.last_heartbeat = Time.zone.at(status_info.timestamp.to_i)
+          db_status.status_type = if status_info.is_a?(Tucana::Shared::AdapterRuntimeStatus)
+                                    :adapter
+                                  else
+                                    :execution
+                                  end
+          db_status.features = status_info.features.to_a
+
+          db_status.status = status_info.status.downcase
+
+          db_configs = db_status.runtime_status_configurations.first(status_info.configurations.size)
+
+          status_info.configurations.each_with_index do |config, index|
+            db_configs[index] ||= db_status.runtime_status_configurations.build
+
+            db_configs[index].endpoint = config.endpoint
+
+            next if db_configs[index].save
+
+            t.rollback_and_return! ServiceResponse.error(
+              message: 'Failed to save runtime status configuration',
+              error_code: :invalid_runtime_status_configuration,
+              details: db_configs.errors
+            )
+          end
+
+          unless db_status.save
+            t.rollback_and_return! ServiceResponse.error(
+              message: 'Failed to save runtime status',
+              error_code: :invalid_runtime_status,
+              details: db_status.errors
+            )
+          end
+
+          return ServiceResponse.success(
+            message: 'Updated runtime status'
+          )
+        end
+      end
+    end
+  end
+end