created functional model_manifest workflow

Author: zeryx

adk/ADK.py

@@ -3,22 +3,24 @@
 import os
 import sys
 from adk.io import create_exception, format_data, format_response
+from adk.manifest import ManifestData
 
 
 class ADK(object):
-    def __init__(self, apply_func, load_func=None):
+    def __init__(self, apply_func, load_func=None, client=None, manifest_path="model_manifest.json.lock"):
         """
         Creates the adk object
         :param apply_func: A required function that can have an arity of 1-2, depending on if loading occurs
         :param load_func: An optional supplier function used if load time events are required, has an arity of 0.
+        :param client: A Algorithmia Client instance that might be user defined, and is used for interacting with a model manifest file; if defined.
         """
         self.FIFO_PATH = "/tmp/algoout"
-        self.manifest_path = "data_manifest.json"
         apply_args, _, _, _, _, _, _ = inspect.getfullargspec(apply_func)
         if load_func:
             load_args, _, _, _, _, _, _ = inspect.getfullargspec(load_func)
-            if len(load_args) > 0:
-                raise Exception("load function must not have parameters")
+            if len(load_args) > 2:
+                raise Exception("load function may either have no parameters, or one parameter providing the manifest "
+                                "state.")
             self.load_func = load_func
         else:
             self.load_func = None
@@ -28,10 +30,14 @@ def __init__(self, apply_func, load_func=None):
         self.is_local = not os.path.exists(self.FIFO_PATH)
         self.load_result = None
         self.loading_exception = None
+        self.manifest = ManifestData(client, manifest_path)
 
     def load(self):
         try:
-            if self.load_func:
+            if self.load_func and self.manifest.available():
+                self.manifest.initialize()
+                self.load_result = self.load_func(self.manifest)
+            elif self.load_func:
                 self.load_result = self.load_func()
         except Exception as e:
             self.loading_exception = e

adk/manifest.py

@@ -3,19 +3,73 @@
 import hashlib
 
 
-def process_manifest(self):
-    if os.path.exists(self.manifest_path):
-        with open(self.manifest_path) as f:
-            manifest_data = json.load(f)
-        manifest = manifest_data
+class ManifestData(object):
+    def __init__(self, client, model_manifest_path):
+        self.manifest_lock_path = model_manifest_path
+        self.manifest_data = get_manifest(self.manifest_lock_path)
+        self.client = client
+        self.models = {}
+
+    def available(self):
+        if self.manifest_data:
+            return True
+        else:
+            return False
+
+    def initialize(self):
+        if self.client is None:
+            raise Exception("Client was not defined, please define a Client when using Model Manifests.")
+        for required_file in self.manifest_data['required_models']:
+            name = required_file['name']
+            if name in self.models:
+                raise Exception("Duplicate 'name' detected. \n"
+                                + name + " was found to be used by more than one data file, please rename.")
+            self.models[name] = {}
+            expected_hash = required_file['md5_checksum']
+            with self.client.file(required_file['data_api_path']).getFile() as f:
+                local_data_path = f.name
+            real_hash = md5(local_data_path)
+            if not real_hash != expected_hash and required_file['fail_on_tamper']:
+                raise Exception("Model File Mismatch for " + name +
+                                "\nexpected hash:  " + expected_hash + "\nreal hash: " + real_hash)
+            else:
+                self.models[name]["md5_checksum"] = real_hash
+                self.models[name]['model_path'] = local_data_path
+
+    def get_model(self, model_name):
+        if model_name in self.models:
+            return self.models[model_name]['model_path']
+        elif model_name in self.manifest_data['optional_files']:
+            self.find_optional_model(model_name)
+            return self.models[model_name]['model_path']
 
+    def find_optional_model(self, model_name):
 
-def check_hash(model_path, expected_hash):
-    real_hash = md5(model_path)
-    if real_hash == expected_hash:
-        return True
+        found_models = [optional for optional in self.manifest_data['optional_models'] if
+                        optional['name'] == model_name]
+        if len(found_models) == 0:
+            raise Exception("model with name '" + model_name + "' not found in model manifest.")
+        model_info = found_models[0]
+        self.models[model_name] = {}
+        expected_hash = model_info['md5_checksum']
+        with self.client.file(model_info['data_api_path']).getFile() as f:
+            local_data_path = f.name
+        real_hash = md5(local_data_path)
+        if not real_hash != expected_hash and model_info['fail_on_tamper']:
+            raise Exception("Model File Mismatch for " + model_name +
+                            "\nexpected hash:  " + expected_hash + "\nreal hash: " + real_hash)
+        else:
+            self.models[model_name]["md5_checksum"] = real_hash
+            self.models[model_name]['model_path'] = local_data_path
+
+
+def get_manifest(manifest_path):
+    if os.path.exists(manifest_path):
+        with open(manifest_path) as f:
+            manifest_data = json.load(f)
+        return manifest_data
     else:
-        return False
+        return None
 
 
 def md5(fname):

examples/pytorch_image_classification/model_manifest.json

@@ -1,19 +1,31 @@
 {
   "algorithm_name": "pytorch_image_classification",
-  "model_versioning": "strict",
-  "required_files" : [
-      { "name": "labels",
+  "required_models": [
+    {
+      "name": "model_squeezenet",
+      "data_api_path": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
+      "fail_on_tamper": true,
+      "metadata": {
+        "origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+      }
+    },
+    {
+      "name": "labels",
       "data_api_path": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
-      "md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
-      "origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+      "fail_on_tamper": true,
+      "metadata": {
+        "origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+      }
     }
   ],
-  "optional_files": [
+  "optional_models": [
     {
-      "name": "optional_data",
-      "data_api_path": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
-      "md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
-      "origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+      "name": "vgg",
+      "data_api_path": "data://AlgorithmiaSE/image_cassification_demo/vgg16-397923af.pth",
+      "fail_on_tamper": false,
+      "metadata": {
+        "origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+      }
     }
   ]
 }

examples/pytorch_image_classification/model_manifest.json.lock

@@ -0,0 +1,25 @@
+{
+  "algorithm_name": "pytorch_image_classification",
+  "model_tamper_behavior": "strict",
+  "lock_version": "f7aec94cde3ca6274c6556ebe58ad3c6",
+  "timestamp": "1632753496"
+  "required_models" : [
+      { "name": "labels",
+      "data_api_path": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
+      "md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
+      "metadata": {
+        "origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+      }
+    }
+  ],
+  "optional_models": [
+    {
+      "name": "optional_data",
+      "data_api_path": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
+      "md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
+      "metadata": {
+        "origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+      }
+    }
+  ]
+}

examples/pytorch_image_classification/src/Algorithm.py

@@ -5,18 +5,19 @@
 import json
 from torchvision import models, transforms
 
-def load_labels(label_path, client):
-    local_path = client.file(label_path).getFile().name
-    with open(local_path) as f:
+
+client = Algorithmia.client()
+
+def load_labels(label_path):
+    with open(label_path) as f:
         labels = json.load(f)
     labels = [labels[str(k)][1] for k in range(len(labels))]
     return labels
 
 
-def load_model(model_paths, client):
+def load_model(model_path):
     model = models.squeezenet1_1()
-    local_file = client.file(model_paths["filepath"]).getFile().name
-    weights = torch.load(local_file)
+    weights = torch.load(model_path)
     model.load_state_dict(weights)
     return model.float().eval()
 
@@ -52,26 +53,25 @@ def infer_image(image_url, n, globals):
 
 def load(manifest):
 
-    globals = {}
-    client = Algorithmia.client()
-    globals["SMID_ALGO"] = "algo://util/SmartImageDownloader/0.2.x"
-    globals["model"] = load_model(manifest["model_squeezenet"], client)
-    globals["labels"] = load_labels(manifest["labels"], client)
-    return globals
+    state = {}
+    state["SMID_ALGO"] = "algo://util/SmartImageDownloader/0.2.x"
+    state["model"] = load_model(manifest.get_model("model_squeezenet"))
+    state["labels"] = load_labels(manifest.get_model("labels"))
+    return state
 
 
-def apply(input, globals):
+def apply(input, state):
     if isinstance(input, dict):
         if "n" in input:
             n = input["n"]
         else:
             n = 3
         if "data" in input:
             if isinstance(input["data"], str):
-                output = infer_image(input["data"], n, globals)
+                output = infer_image(input["data"], n, state)
             elif isinstance(input["data"], list):
                 for row in input["data"]:
-                    row["predictions"] = infer_image(row["image_url"], n, globals)
+                    row["predictions"] = infer_image(row["image_url"], n, state)
                 output = input["data"]
             else:
                 raise Exception("\"data\" must be a image url or a list of image urls (with labels)")
@@ -82,5 +82,5 @@ def apply(input, globals):
         raise Exception("input must be a json object")
 
 
-algorithm = ADK(apply_func=apply, load_func=load)
+algorithm = ADK(apply_func=apply, load_func=load, client=client)
 algorithm.init({"data": "https://i.imgur.com/bXdORXl.jpeg"})