added lock file tamper detection

Author: zeryx

adk/ADK.py

@@ -13,6 +13,7 @@ def __init__(self, apply_func, load_func=None, client=None, manifest_path="model
         :param apply_func: A required function that can have an arity of 1-2, depending on if loading occurs
         :param load_func: An optional supplier function used if load time events are required, has an arity of 0.
         :param client: A Algorithmia Client instance that might be user defined, and is used for interacting with a model manifest file; if defined.
+        :param manifest_path: A development / testing facing variable used to set the name and path
         """
         self.FIFO_PATH = "/tmp/algoout"
         apply_args, _, _, _, _, _, _ = inspect.getfullargspec(apply_func)

adk/manifest.py

@@ -28,7 +28,7 @@ def initialize(self):
             expected_hash = required_file['md5_checksum']
             with self.client.file(required_file['data_api_path']).getFile() as f:
                 local_data_path = f.name
-            real_hash = md5(local_data_path)
+            real_hash = md5_for_file(local_data_path)
             if real_hash != expected_hash and required_file['fail_on_tamper']:
                 raise Exception("Model File Mismatch for " + name +
                                 "\nexpected hash:  " + expected_hash + "\nreal hash: " + real_hash)
@@ -57,7 +57,7 @@ def find_optional_model(self, model_name):
         expected_hash = model_info['md5_checksum']
         with self.client.file(model_info['data_api_path']).getFile() as f:
             local_data_path = f.name
-        real_hash = md5(local_data_path)
+        real_hash = md5_for_file(local_data_path)
         if real_hash != expected_hash and model_info['fail_on_tamper']:
             raise Exception("Model File Mismatch for " + model_name +
                             "\nexpected hash:  " + expected_hash + "\nreal hash: " + real_hash)
@@ -70,14 +70,26 @@ def get_manifest(manifest_path):
     if os.path.exists(manifest_path):
         with open(manifest_path) as f:
             manifest_data = json.load(f)
+        expected_lock_checksum = manifest_data.get('lock_checksum')
+        del manifest_data['lock_checksum']
+        detected_lock_checksum = md5_for_str(str(manifest_data))
+        if expected_lock_checksum != detected_lock_checksum:
+            raise Exception("Manifest Lockfile Tamper Detected; please use the CLI and 'algo compile' to rebuild your "
+                            "algorithm's lock file.")
         return manifest_data
     else:
         return None
 
 
-def md5(fname):
+def md5_for_file(fname):
     hash_md5 = hashlib.md5()
     with open(fname, "rb") as f:
         for chunk in iter(lambda: f.read(4096), b""):
             hash_md5.update(chunk)
     return str(hash_md5.hexdigest())
+
+
+def md5_for_str(content):
+    hash_md5 = hashlib.md5()
+    hash_md5.update(content.encode())
+    return str(hash_md5.hexdigest())

tests/bad_model_manifest.json.lock …s/manifests/bad_model_manifest.json.locktests/bad_model_manifest.json.lock renamed to tests/manifests/bad_model_manifest.json.lock tests/bad_model_manifest.json.lock renamed to tests/manifests/bad_model_manifest.json.lock

@@ -1,6 +1,7 @@
 {
   "algorithm_name": "test_algorithm",
   "timestamp": "1632770803",
+  "lock_checksum": "57dcf8ab156f0c86f2f275919dcbf090",
   "required_models" : [
       { "name": "squeezenet",
       "data_api_path": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",

tests/good_model_manifest.json.lock …/manifests/good_model_manifest.json.locktests/good_model_manifest.json.lock renamed to tests/manifests/good_model_manifest.json.lock tests/good_model_manifest.json.lock renamed to tests/manifests/good_model_manifest.json.lock

@@ -1,6 +1,7 @@
 {
   "algorithm_name": "test_algorithm",
   "timestamp": "1632770803",
+  "lock_checksum": "fba73f61886f0921b47997057e853d36",
   "required_models" : [
       { "name": "squeezenet",
       "data_api_path": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",

tests/test_adk_local.py

@@ -20,7 +20,7 @@ def execute_example(self, input, apply, load=lambda: None):
         algo.init(input, pprint=lambda x: output.append(x))
         return output[0]
 
-    def execute_manifest_example(self, input, apply, load, manifest_path="good_model_manifest.json.lock"):
+    def execute_manifest_example(self, input, apply, load, manifest_path="manifests/good_model_manifest.json.lock"):
         client = Algorithmia.client()
         algo = ADK(apply, load, manifest_path=manifest_path, client=client)
         output = []
@@ -127,7 +127,8 @@ def test_manifest_file_success(self):
         }
         actual_output = json.loads(self.execute_manifest_example(input, apply_successful_manifest_parsing,
                                                                  loading_with_manifest,
-                                                                 manifest_path="tests/good_model_manifest.json.lock"))
+                                                                 manifest_path="tests/manifests/good_model_manifest"
+                                                                               ".json.lock"))
         self.assertEqual(expected_output, actual_output)
 
     def test_manifest_file_tampered(self):
@@ -140,7 +141,8 @@ def test_manifest_file_tampered(self):
 
         actual_output = json.loads(self.execute_manifest_example(input, apply_successful_manifest_parsing,
                                                                  loading_with_manifest,
-                                                                 manifest_path="tests/bad_model_manifest.json.lock"))
+                                                                 manifest_path="tests/manifests/bad_model_manifest"
+                                                                               ".json.lock"))
         self.assertEqual(expected_output, actual_output)