diff --git a/Cargo.lock b/Cargo.lock index bb46d46d6..e17c082f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2952,7 +2952,7 @@ dependencies = [ [[package]] name = "magicblock-delegation-program" version = "1.1.2" -source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=e8d03936#e8d039369ac1149e899ea94f31e0f9cc4e600a38" +source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=ea1f2f916268132248fe8d5de5f07d76765dd937#ea1f2f916268132248fe8d5de5f07d76765dd937" dependencies = [ "bincode", "borsh 1.5.7", diff --git a/Cargo.toml b/Cargo.toml index 7662b8f7d..6fe549e3c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,7 +95,7 @@ magicblock-committor-program = { path = "./magicblock-committor-program", featur magicblock-committor-service = { path = "./magicblock-committor-service" } magicblock-config = { path = "./magicblock-config" } magicblock-core = { path = "./magicblock-core" } -magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "e8d03936", features = [ +magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "ea1f2f916268132248fe8d5de5f07d76765dd937", features = [ "no-entrypoint", ] } magicblock-geyser-plugin = { path = "./magicblock-geyser-plugin" } diff --git a/magicblock-committor-service/src/tasks/args_task.rs b/magicblock-committor-service/src/tasks/args_task.rs index a7e6ca81f..c5a942319 100644 --- a/magicblock-committor-service/src/tasks/args_task.rs +++ b/magicblock-committor-service/src/tasks/args_task.rs @@ -132,16 +132,8 @@ impl BaseTask for ArgsTask { ))) } ArgsTaskType::CommitDiff(value) => { - // TODO (snawaz): Currently, we do not support executing CommitDiff - // as BufferTask, which is why we're forcing CommitDiffTask to become CommitTask - // before converting this task into BufferTask. Once CommitDiff is supported - // by BufferTask, we do not have to do this, as it's essentially a downgrade. Ok(Box::new(BufferTask::new_preparation_required( - BufferTaskType::Commit(CommitTask { - commit_id: value.commit_id, - allow_undelegation: value.allow_undelegation, - committed_account: value.committed_account, - }), + BufferTaskType::CommitDiff(value), ))) } ArgsTaskType::BaseAction(_) diff --git a/magicblock-committor-service/src/tasks/buffer_task.rs b/magicblock-committor-service/src/tasks/buffer_task.rs index ccb54b715..d5dfaa966 100644 --- a/magicblock-committor-service/src/tasks/buffer_task.rs +++ b/magicblock-committor-service/src/tasks/buffer_task.rs @@ -1,4 +1,4 @@ -use dlp::args::CommitStateFromBufferArgs; +use dlp::{args::CommitStateFromBufferArgs, compute_diff}; use magicblock_committor_program::Chunks; use magicblock_metrics::metrics::LabelValue; use solana_instruction::Instruction; @@ -9,8 +9,9 @@ use crate::tasks::TaskStrategy; use crate::{ consts::MAX_WRITE_CHUNK_SIZE, tasks::{ - visitor::Visitor, BaseTask, BaseTaskError, BaseTaskResult, CommitTask, - PreparationState, PreparationTask, TaskType, + visitor::Visitor, BaseTask, BaseTaskError, BaseTaskResult, + CommitDiffTask, CommitTask, PreparationState, PreparationTask, + TaskType, }, }; @@ -18,6 +19,7 @@ use crate::{ #[derive(Clone)] pub enum BufferTaskType { Commit(CommitTask), + CommitDiff(CommitDiffTask), // Action in the future } @@ -46,50 +48,94 @@ impl BufferTask { } fn preparation_required(task_type: &BufferTaskType) -> PreparationState { - let BufferTaskType::Commit(ref commit_task) = task_type; - let committed_data = commit_task.committed_account.account.data.clone(); - let chunks = Chunks::from_data_length( - committed_data.len(), - MAX_WRITE_CHUNK_SIZE, - ); - - PreparationState::Required(PreparationTask { - commit_id: commit_task.commit_id, - pubkey: commit_task.committed_account.pubkey, - committed_data, - chunks, - }) + match task_type { + BufferTaskType::Commit(task) => { + let data = task.committed_account.account.data.clone(); + let chunks = + Chunks::from_data_length(data.len(), MAX_WRITE_CHUNK_SIZE); + + PreparationState::Required(PreparationTask { + commit_id: task.commit_id, + pubkey: task.committed_account.pubkey, + committed_data: data, + chunks, + }) + } + + BufferTaskType::CommitDiff(task) => { + let diff = compute_diff( + &task.committed_account.account.data, + &task.base_account.data, + ) + .to_vec(); + let chunks = + Chunks::from_data_length(diff.len(), MAX_WRITE_CHUNK_SIZE); + + PreparationState::Required(PreparationTask { + commit_id: task.commit_id, + pubkey: task.committed_account.pubkey, + committed_data: diff, + chunks, + }) + } + } } } impl BaseTask for BufferTask { fn instruction(&self, validator: &Pubkey) -> Instruction { - let BufferTaskType::Commit(ref value) = self.task_type; - let commit_id_slice = value.commit_id.to_le_bytes(); - let (commit_buffer_pubkey, _) = - magicblock_committor_program::pdas::buffer_pda( - validator, - &value.committed_account.pubkey, - &commit_id_slice, - ); - - dlp::instruction_builder::commit_state_from_buffer( - *validator, - value.committed_account.pubkey, - value.committed_account.account.owner, - commit_buffer_pubkey, - CommitStateFromBufferArgs { - nonce: value.commit_id, - lamports: value.committed_account.account.lamports, - allow_undelegation: value.allow_undelegation, - }, - ) + match &self.task_type { + BufferTaskType::Commit(task) => { + let commit_id_slice = task.commit_id.to_le_bytes(); + let (commit_buffer_pubkey, _) = + magicblock_committor_program::pdas::buffer_pda( + validator, + &task.committed_account.pubkey, + &commit_id_slice, + ); + + dlp::instruction_builder::commit_state_from_buffer( + *validator, + task.committed_account.pubkey, + task.committed_account.account.owner, + commit_buffer_pubkey, + CommitStateFromBufferArgs { + nonce: task.commit_id, + lamports: task.committed_account.account.lamports, + allow_undelegation: task.allow_undelegation, + }, + ) + } + BufferTaskType::CommitDiff(task) => { + let commit_id_slice = task.commit_id.to_le_bytes(); + let (commit_buffer_pubkey, _) = + magicblock_committor_program::pdas::buffer_pda( + validator, + &task.committed_account.pubkey, + &commit_id_slice, + ); + + dlp::instruction_builder::commit_diff_from_buffer( + *validator, + task.committed_account.pubkey, + task.committed_account.account.owner, + commit_buffer_pubkey, + CommitStateFromBufferArgs { + nonce: task.commit_id, + lamports: task.committed_account.account.lamports, + allow_undelegation: task.allow_undelegation, + }, + ) + } + } } /// No further optimizations fn optimize( self: Box, ) -> Result, Box> { + // Since the buffer in BufferTask doesn't contribute to the size of + // transaction, there is nothing we can do here to optimize/reduce the size. Err(self) } @@ -112,6 +158,7 @@ impl BaseTask for BufferTask { fn compute_units(&self) -> u32 { match self.task_type { BufferTaskType::Commit(_) => 70_000, + BufferTaskType::CommitDiff(_) => 70_000, } } @@ -123,6 +170,7 @@ impl BaseTask for BufferTask { fn task_type(&self) -> TaskType { match self.task_type { BufferTaskType::Commit(_) => TaskType::Commit, + BufferTaskType::CommitDiff(_) => TaskType::Commit, } } @@ -132,12 +180,21 @@ impl BaseTask for BufferTask { } fn reset_commit_id(&mut self, commit_id: u64) { - let BufferTaskType::Commit(commit_task) = &mut self.task_type; - if commit_id == commit_task.commit_id { - return; - } + match &mut self.task_type { + BufferTaskType::Commit(commit_task) => { + if commit_id == commit_task.commit_id { + return; + } + commit_task.commit_id = commit_id; + } + BufferTaskType::CommitDiff(task) => { + if commit_id == task.commit_id { + return; + } + task.commit_id = commit_id; + } + }; - commit_task.commit_id = commit_id; self.preparation_state = Self::preparation_required(&self.task_type) } } @@ -146,6 +203,7 @@ impl LabelValue for BufferTask { fn value(&self) -> &str { match self.task_type { BufferTaskType::Commit(_) => "buffer_commit", + BufferTaskType::CommitDiff(_) => "buffer_commit_diff", } } } diff --git a/magicblock-committor-service/src/tasks/mod.rs b/magicblock-committor-service/src/tasks/mod.rs index 037c6dbf4..457811c36 100644 --- a/magicblock-committor-service/src/tasks/mod.rs +++ b/magicblock-committor-service/src/tasks/mod.rs @@ -44,7 +44,6 @@ pub enum PreparationState { Cleanup(CleanupTask), } -#[cfg(test)] #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub enum TaskStrategy { Args, diff --git a/magicblock-committor-service/src/tasks/task_visitors/persistor_visitor.rs b/magicblock-committor-service/src/tasks/task_visitors/persistor_visitor.rs index c608f2ef9..347f67959 100644 --- a/magicblock-committor-service/src/tasks/task_visitors/persistor_visitor.rs +++ b/magicblock-committor-service/src/tasks/task_visitors/persistor_visitor.rs @@ -26,27 +26,40 @@ where fn visit_args_task(&mut self, task: &ArgsTask) { match self.context { PersistorContext::PersistStrategy { uses_lookup_tables } => { - let ArgsTaskType::Commit(ref commit_task) = task.task_type - else { - return; - }; - let commit_strategy = if uses_lookup_tables { CommitStrategy::ArgsWithLookupTable } else { CommitStrategy::Args }; - if let Err(err) = self.persistor.set_commit_strategy( - commit_task.commit_id, - &commit_task.committed_account.pubkey, - commit_strategy, - ) { - error!( - "Failed to persist commit strategy {}: {}", - commit_strategy.as_str(), - err - ); + match &task.task_type { + ArgsTaskType::Commit(task) => { + if let Err(err) = self.persistor.set_commit_strategy( + task.commit_id, + &task.committed_account.pubkey, + commit_strategy, + ) { + error!( + "Failed to persist commit strategy {}: {}", + commit_strategy.as_str(), + err + ); + } + } + ArgsTaskType::CommitDiff(task) => { + if let Err(err) = self.persistor.set_commit_strategy( + task.commit_id, + &task.committed_account.pubkey, + commit_strategy, + ) { + error!( + "Failed to persist commit strategy {}: {}", + commit_strategy.as_str(), + err + ); + } + } + _ => {} } } } @@ -55,23 +68,39 @@ where fn visit_buffer_task(&mut self, task: &BufferTask) { match self.context { PersistorContext::PersistStrategy { uses_lookup_tables } => { - let BufferTaskType::Commit(ref commit_task) = task.task_type; let commit_strategy = if uses_lookup_tables { CommitStrategy::FromBufferWithLookupTable } else { CommitStrategy::FromBuffer }; - if let Err(err) = self.persistor.set_commit_strategy( - commit_task.commit_id, - &commit_task.committed_account.pubkey, - commit_strategy, - ) { - error!( - "Failed to persist commit strategy {}: {}", - commit_strategy.as_str(), - err - ); + match &task.task_type { + BufferTaskType::Commit(task) => { + if let Err(err) = self.persistor.set_commit_strategy( + task.commit_id, + &task.committed_account.pubkey, + commit_strategy, + ) { + error!( + "Failed to persist commit strategy {}: {}", + commit_strategy.as_str(), + err + ); + } + } + BufferTaskType::CommitDiff(task) => { + if let Err(err) = self.persistor.set_commit_strategy( + task.commit_id, + &task.committed_account.pubkey, + commit_strategy, + ) { + error!( + "Failed to persist commit strategy {}: {}", + commit_strategy.as_str(), + err + ); + } + } } } } diff --git a/magicblock-committor-service/src/tasks/task_visitors/utility_visitor.rs b/magicblock-committor-service/src/tasks/task_visitors/utility_visitor.rs index fef7cade1..470e25341 100644 --- a/magicblock-committor-service/src/tasks/task_visitors/utility_visitor.rs +++ b/magicblock-committor-service/src/tasks/task_visitors/utility_visitor.rs @@ -19,23 +19,39 @@ impl Visitor for TaskVisitorUtils { fn visit_args_task(&mut self, task: &ArgsTask) { let Self::GetCommitMeta(commit_meta) = self; - if let ArgsTaskType::Commit(ref commit_task) = task.task_type { - *commit_meta = Some(CommitMeta { - committed_pubkey: commit_task.committed_account.pubkey, - commit_id: commit_task.commit_id, - }) - } else { - *commit_meta = None + match &task.task_type { + ArgsTaskType::Commit(task) => { + *commit_meta = Some(CommitMeta { + committed_pubkey: task.committed_account.pubkey, + commit_id: task.commit_id, + }) + } + ArgsTaskType::CommitDiff(task) => { + *commit_meta = Some(CommitMeta { + committed_pubkey: task.committed_account.pubkey, + commit_id: task.commit_id, + }) + } + _ => *commit_meta = None, } } fn visit_buffer_task(&mut self, task: &BufferTask) { let Self::GetCommitMeta(commit_meta) = self; - let BufferTaskType::Commit(ref commit_task) = task.task_type; - *commit_meta = Some(CommitMeta { - committed_pubkey: commit_task.committed_account.pubkey, - commit_id: commit_task.commit_id, - }) + match &task.task_type { + BufferTaskType::Commit(task) => { + *commit_meta = Some(CommitMeta { + committed_pubkey: task.committed_account.pubkey, + commit_id: task.commit_id, + }) + } + BufferTaskType::CommitDiff(task) => { + *commit_meta = Some(CommitMeta { + committed_pubkey: task.committed_account.pubkey, + commit_id: task.commit_id, + }) + } + } } } diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 00ad9c90d..fb8452c27 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -3362,7 +3362,7 @@ dependencies = [ [[package]] name = "magicblock-delegation-program" version = "1.1.2" -source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=e8d03936#e8d039369ac1149e899ea94f31e0f9cc4e600a38" +source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=ea1f2f916268132248fe8d5de5f07d76765dd937#ea1f2f916268132248fe8d5de5f07d76765dd937" dependencies = [ "bincode", "borsh 1.5.7", diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index 23df65f67..260e9b932 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -57,8 +57,8 @@ magicblock-config = { path = "../magicblock-config" } magicblock-core = { path = "../magicblock-core" } magic-domain-program = { git = "https://github.com/magicblock-labs/magic-domain-program.git", rev = "ea04d46", default-features = false } magicblock_magic_program_api = { package = "magicblock-magic-program-api", path = "../magicblock-magic-program-api" } -magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "e8d03936", features = [ - "no-entrypoint", +magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "ea1f2f916268132248fe8d5de5f07d76765dd937", features = [ + "no-entrypoint", ] } magicblock-program = { path = "../programs/magicblock" } magicblock-rpc-client = { path = "../magicblock-rpc-client" } diff --git a/test-integration/schedulecommit/elfs/dlp.so b/test-integration/schedulecommit/elfs/dlp.so index decfd0f00..46fe9a4d1 100755 Binary files a/test-integration/schedulecommit/elfs/dlp.so and b/test-integration/schedulecommit/elfs/dlp.so differ diff --git a/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs b/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs index daaea5506..bf7412986 100644 --- a/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs +++ b/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs @@ -303,13 +303,13 @@ fn test_committing_and_undelegating_huge_order_book_account() { println!("Important: use {rng_seed} as seed to regenerate the random inputs in case of test failure"); let mut random = StdRng::seed_from_u64(rng_seed); let mut update = BookUpdate::default(); - update.bids.extend((0..random.gen_range(5..10)).map(|_| { + update.bids.extend((0..random.gen_range(5..100)).map(|_| { OrderLevel { price: random.gen_range(75000..90000), size: random.gen_range(1..10), } })); - update.asks.extend((0..random.gen_range(5..10)).map(|_| { + update.asks.extend((0..random.gen_range(5..100)).map(|_| { OrderLevel { price: random.gen_range(125000..150000), size: random.gen_range(1..10),