From 2d7b46e90c5927ef34ad79bc9290563125f3de60 Mon Sep 17 00:00:00 2001 From: Arpan Saini Date: Fri, 24 May 2024 00:18:17 +0530 Subject: [PATCH 1/4] CP-40707 Sync Upload for S3/R2. --- classes/local/manager.php | 9 +++++++++ .../candidates/deleter_candidates.php | 3 ++- .../candidates/pusher_candidates.php | 3 ++- .../local/object_manipulator/manipulator_builder.php | 11 ++++++----- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/classes/local/manager.php b/classes/local/manager.php index c746b9c2..46fd243e 100644 --- a/classes/local/manager.php +++ b/classes/local/manager.php @@ -27,6 +27,9 @@ use stdClass; use tool_objectfs\local\store\object_file_system; +use tool_objectfs\local\object_manipulator\manipulator_builder; +use tool_objectfs\local\object_manipulator\pusher; +use tool_objectfs\local\object_manipulator\deleter; defined('MOODLE_INTERNAL') || die(); @@ -168,6 +171,12 @@ public static function update_object_by_hash($contenthash, $newlocation, $filesi } $DB->insert_record('tool_objectfs_objects', $newobject); + $manipulator_builder = new manipulator_builder(); + $extraconfig = new stdClass(); + $extraconfig->sqlfilterstring = "contenthash = '{$contenthash}'"; + $manipulator_builder->execute(pusher::class, $extraconfig); + $manipulator_builder->execute(deleter::class, $extraconfig); + return $newobject; } diff --git a/classes/local/object_manipulator/candidates/deleter_candidates.php b/classes/local/object_manipulator/candidates/deleter_candidates.php index e17a179e..cedb8798 100644 --- a/classes/local/object_manipulator/candidates/deleter_candidates.php +++ b/classes/local/object_manipulator/candidates/deleter_candidates.php @@ -39,7 +39,8 @@ public function get_candidates_sql() { FROM {tool_objectfs_objects} WHERE timeduplicated <= :consistancythreshold AND location = :location - AND filesize > :sizethreshold'; + AND filesize > :sizethreshold' . ( + empty($this->config->extraconfig->sqlfilterstring) ? "" : " AND " . $this->config->extraconfig->sqlfilterstring); } /** diff --git a/classes/local/object_manipulator/candidates/pusher_candidates.php b/classes/local/object_manipulator/candidates/pusher_candidates.php index a685a975..fe07b095 100644 --- a/classes/local/object_manipulator/candidates/pusher_candidates.php +++ b/classes/local/object_manipulator/candidates/pusher_candidates.php @@ -42,7 +42,8 @@ public function get_candidates_sql() { WHERE filesize > :threshold AND filesize < :maximum_file_size AND timeduplicated <= :maxcreatedtimestamp - AND location = :object_location'; + AND location = :object_location' . ( + empty($this->config->extraconfig->sqlfilterstring) ? "" : " AND " . $this->config->extraconfig->sqlfilterstring); } /** diff --git a/classes/local/object_manipulator/manipulator_builder.php b/classes/local/object_manipulator/manipulator_builder.php index 66e5ba41..7bb5cf30 100644 --- a/classes/local/object_manipulator/manipulator_builder.php +++ b/classes/local/object_manipulator/manipulator_builder.php @@ -68,8 +68,8 @@ class manipulator_builder { * @throws coding_exception * @throws moodle_exception */ - public function execute($manipulator) { - $this->build($manipulator); + public function execute($manipulator, $extraconfig = null) { + $this->build($manipulator, $extraconfig); if (empty($this->candidates)) { return; } @@ -86,10 +86,10 @@ public function execute($manipulator) { * @throws coding_exception * @throws moodle_exception */ - public function execute_all() { + public function execute_all($extraconfig = null) { foreach ($this->manipulators as $manipulator) { mtrace("Executing objectfs $manipulator"); - $this->execute($manipulator); + $this->execute($manipulator, $extraconfig); mtrace("Objectfs $manipulator successfully executed"); } } @@ -98,8 +98,9 @@ public function execute_all() { * @param string $manipulator * @throws moodle_exception */ - private function build($manipulator) { + private function build($manipulator, $extraconfig = null) { $this->config = manager::get_objectfs_config(); + $this->config->extraconfig = $extraconfig; $this->manipulatorclass = $manipulator; $this->logger = new aggregate_logger(); $this->finder = new candidates_finder($manipulator, $this->config); From 6539384f536ed7e10f92ffbc45b8d49f5e719433 Mon Sep 17 00:00:00 2001 From: Arpan Saini Date: Wed, 29 May 2024 13:27:11 +0530 Subject: [PATCH 2/4] Remove Deleter/Pusher Logs --- .../local/object_manipulator/manipulator.php | 2 +- classes/local/store/object_file_system.php | 20 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/classes/local/object_manipulator/manipulator.php b/classes/local/object_manipulator/manipulator.php index a0cadaf8..df11f04d 100644 --- a/classes/local/object_manipulator/manipulator.php +++ b/classes/local/object_manipulator/manipulator.php @@ -115,7 +115,7 @@ public function execute(array $objectrecords) { } $this->logger->end_timing(); - $this->logger->output_move_statistics(); + // $this->logger->output_move_statistics(); } /** diff --git a/classes/local/store/object_file_system.php b/classes/local/store/object_file_system.php index 10692f1a..195a828f 100644 --- a/classes/local/store/object_file_system.php +++ b/classes/local/store/object_file_system.php @@ -267,11 +267,11 @@ public function copy_object_from_local_to_external_by_hash($contenthash, $object } } - $this->logger->log_object_move('copy_object_from_local_to_external', - $initiallocation, - $finallocation, - $contenthash, - $objectsize); + // $this->logger->log_object_move('copy_object_from_local_to_external', + // $initiallocation, + // $finallocation, + // $contenthash, + // $objectsize); return $finallocation; } @@ -296,11 +296,11 @@ public function delete_object_from_local_by_hash($contenthash, $objectsize = 0) } } - $this->logger->log_object_move('delete_local_object', - $initiallocation, - $finallocation, - $contenthash, - $objectsize); + // $this->logger->log_object_move('delete_local_object', + // $initiallocation, + // $finallocation, + // $contenthash, + // $objectsize); return $finallocation; } From e2db193ecf30fb58b53f7600276180d35eb6e2e1 Mon Sep 17 00:00:00 2001 From: Arpan Saini Date: Thu, 30 May 2024 10:26:53 +0530 Subject: [PATCH 3/4] Hide bucket key prefix from UI --- classes/local/store/s3/client.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/classes/local/store/s3/client.php b/classes/local/store/s3/client.php index 31288859..eb396fcc 100644 --- a/classes/local/store/s3/client.php +++ b/classes/local/store/s3/client.php @@ -422,9 +422,9 @@ public function define_client_section($settings, $config) { new \lang_string('settings:aws:base_url', 'tool_objectfs'), new \lang_string('settings:aws:base_url_help', 'tool_objectfs'), '')); - $settings->add(new \admin_setting_configtext('tool_objectfs/key_prefix', - new \lang_string('settings:aws:key_prefix', 'tool_objectfs'), - new \lang_string('settings:aws:key_prefix_help', 'tool_objectfs'), '')); + // $settings->add(new \admin_setting_configtext('tool_objectfs/key_prefix', + // new \lang_string('settings:aws:key_prefix', 'tool_objectfs'), + // new \lang_string('settings:aws:key_prefix_help', 'tool_objectfs'), '')); return $settings; } From 9a82449d901c45abbdfd6539e8ec7938eca960a3 Mon Sep 17 00:00:00 2001 From: Arpan Saini Date: Thu, 30 May 2024 16:13:24 +0530 Subject: [PATCH 4/4] Add bucketkeyprefix to s3 client --- classes/local/store/s3/client.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/classes/local/store/s3/client.php b/classes/local/store/s3/client.php index eb396fcc..a464dd0f 100644 --- a/classes/local/store/s3/client.php +++ b/classes/local/store/s3/client.php @@ -47,6 +47,7 @@ class client extends object_client_base { protected $client; protected $bucket; private $signingmethod; + private $bucketkeyprefix; public function __construct($config) { global $CFG; @@ -422,9 +423,9 @@ public function define_client_section($settings, $config) { new \lang_string('settings:aws:base_url', 'tool_objectfs'), new \lang_string('settings:aws:base_url_help', 'tool_objectfs'), '')); - // $settings->add(new \admin_setting_configtext('tool_objectfs/key_prefix', - // new \lang_string('settings:aws:key_prefix', 'tool_objectfs'), - // new \lang_string('settings:aws:key_prefix_help', 'tool_objectfs'), '')); + $settings->add(new \admin_setting_configtext('tool_objectfs/key_prefix', + new \lang_string('settings:aws:key_prefix', 'tool_objectfs'), + new \lang_string('settings:aws:key_prefix_help', 'tool_objectfs'), '')); return $settings; }