initial commit, updated example to spec

Author: zeryx

adk/ADK.py

@@ -1,10 +1,8 @@
-import base64
 import inspect
 import json
 import os
 import sys
-import traceback
-import six
+from adk.io import create_exception, format_data, format_response
 
 
 class ADK(object):
@@ -15,6 +13,7 @@ def __init__(self, apply_func, load_func=None):
         :param load_func: An optional supplier function used if load time events are required, has an arity of 0.
         """
         self.FIFO_PATH = "/tmp/algoout"
+        self.manifest_path = "data_manifest.json"
         apply_args, _, _, _, _, _, _ = inspect.getfullargspec(apply_func)
         if load_func:
             load_args, _, _, _, _, _, _ = inspect.getfullargspec(load_func)
@@ -49,51 +48,12 @@ def apply(self, payload):
                 apply_result = self.apply_func(payload, self.load_result)
             else:
                 apply_result = self.apply_func(payload)
-            response_obj = self.format_response(apply_result)
+            response_obj = format_response(apply_result)
             return response_obj
         except Exception as e:
-            response_obj = self.create_exception(e)
+            response_obj = create_exception(e)
             return response_obj
 
-    def format_data(self, request):
-        if request["content_type"] in ["text", "json"]:
-            data = request["data"]
-        elif request["content_type"] == "binary":
-            data = self.wrap_binary_data(base64.b64decode(request["data"]))
-        else:
-            raise Exception("Invalid content_type: {}".format(request["content_type"]))
-        return data
-
-    def is_binary(self, arg):
-        if six.PY3:
-            return isinstance(arg, base64.bytes_types)
-
-        return isinstance(arg, bytearray)
-
-    def wrap_binary_data(self, data):
-        if six.PY3:
-            return bytes(data)
-        else:
-            return bytearray(data)
-
-    def format_response(self, response):
-        if self.is_binary(response):
-            content_type = "binary"
-            response = str(base64.b64encode(response), "utf-8")
-        elif isinstance(response, six.string_types) or isinstance(response, six.text_type):
-            content_type = "text"
-        else:
-            content_type = "json"
-        response_string = json.dumps(
-            {
-                "result": response,
-                "metadata": {
-                    "content_type": content_type
-                }
-            }
-        )
-        return response_string
-
     def write_to_pipe(self, payload, pprint=print):
         if self.is_local:
             if isinstance(payload, dict):
@@ -109,40 +69,24 @@ def write_to_pipe(self, payload, pprint=print):
             if os.name == "nt":
                 sys.stdin = payload
 
-    def create_exception(self, exception, loading_exception=False):
-        if hasattr(exception, "error_type"):
-            error_type = exception.error_type
-        elif loading_exception:
-            error_type = "LoadingError"
-        else:
-            error_type = "AlgorithmError"
-        response = json.dumps({
-            "error": {
-                "message": str(exception),
-                "stacktrace": traceback.format_exc(),
-                "error_type": error_type,
-            }
-        })
-        return response
-
     def process_local(self, local_payload, pprint):
         result = self.apply(local_payload)
         self.write_to_pipe(result, pprint=pprint)
 
     def init(self, local_payload=None, pprint=print):
-            self.load()
-            if self.is_local and local_payload:
+        self.load()
+        if self.is_local and local_payload:
+            if self.loading_exception:
+                load_error = create_exception(self.loading_exception, loading_exception=True)
+                self.write_to_pipe(load_error, pprint=pprint)
+            self.process_local(local_payload, pprint)
+        else:
+            for line in sys.stdin:
+                request = json.loads(line)
+                formatted_input = format_data(request)
                 if self.loading_exception:
-                    load_error = self.create_exception(self.loading_exception, loading_exception=True)
+                    load_error = create_exception(self.loading_exception, loading_exception=True)
                     self.write_to_pipe(load_error, pprint=pprint)
-                self.process_local(local_payload, pprint)
-            else:
-                for line in sys.stdin:
-                    request = json.loads(line)
-                    formatted_input = self.format_data(request)
-                    if self.loading_exception:
-                        load_error = self.create_exception(self.loading_exception, loading_exception=True)
-                        self.write_to_pipe(load_error, pprint=pprint)
-                    else:
-                        result = self.apply(formatted_input)
-                        self.write_to_pipe(result)
+                else:
+                    result = self.apply(formatted_input)
+                    self.write_to_pipe(result)

adk/io.py

@@ -0,0 +1,64 @@
+import traceback
+import six
+import base64
+import json
+
+
+def format_data(request):
+    if request["content_type"] in ["text", "json"]:
+        data = request["data"]
+    elif request["content_type"] == "binary":
+        data = wrap_binary_data(base64.b64decode(request["data"]))
+    else:
+        raise Exception("Invalid content_type: {}".format(request["content_type"]))
+    return data
+
+
+def is_binary(arg):
+    if six.PY3:
+        return isinstance(arg, base64.bytes_types)
+
+    return isinstance(arg, bytearray)
+
+
+def wrap_binary_data(data):
+    if six.PY3:
+        return bytes(data)
+    else:
+        return bytearray(data)
+
+
+def format_response(response):
+    if is_binary(response):
+        content_type = "binary"
+        response = str(base64.b64encode(response), "utf-8")
+    elif isinstance(response, six.string_types) or isinstance(response, six.text_type):
+        content_type = "text"
+    else:
+        content_type = "json"
+    response_string = json.dumps(
+        {
+            "result": response,
+            "metadata": {
+                "content_type": content_type
+            }
+        }
+    )
+    return response_string
+
+
+def create_exception(exception, loading_exception=False):
+    if hasattr(exception, "error_type"):
+        error_type = exception.error_type
+    elif loading_exception:
+        error_type = "LoadingError"
+    else:
+        error_type = "AlgorithmError"
+    response = json.dumps({
+        "error": {
+            "message": str(exception),
+            "stacktrace": traceback.format_exc(),
+            "error_type": error_type,
+        }
+    })
+    return response

adk/manifest.py

@@ -0,0 +1,26 @@
+import os
+import json
+import hashlib
+
+
+def process_manifest(self):
+    if os.path.exists(self.manifest_path):
+        with open(self.manifest_path) as f:
+            manifest_data = json.load(f)
+        manifest = manifest_data
+
+
+def check_hash(model_path, expected_hash):
+    real_hash = md5(model_path)
+    if real_hash == expected_hash:
+        return True
+    else:
+        return False
+
+
+def md5(fname):
+    hash_md5 = hashlib.md5()
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()

examples/pytorch_image_classification/model_manifest.json

@@ -1,14 +1,19 @@
 {
-  "label_file": {
-    "filepath": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
-    "md5_hash": "c2c37ea517e94d9795004a39431a14cb",
-    "origin_ref": "this file came from imagenet.org",
-    "uploaded_utc": "2021-05-03-11:05"
-  },
-    "squeezenet": {
-      "filepath": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
-      "md5_hash": "46a44d32d2c5c07f7f66324bef4c7266",
-      "origin_ref": "From https://download.pytorch.org/models/squeezenet1_1-f364aa15.pth",
-      "uploaded_utc": "2021-05-03-11:05"
+  "algorithm_name": "pytorch_image_classification",
+  "model_versioning": "strict",
+  "required_files" : [
+      { "name": "labels",
+      "data_api_path": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
+      "md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
+      "origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
     }
+  ],
+  "optional_files": [
+    {
+      "name": "optional_data",
+      "data_api_path": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
+      "md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
+      "origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
+    }
+  ]
 }

examples/pytorch_image_classification/src/Algorithm.py

@@ -55,8 +55,8 @@ def load(manifest):
     globals = {}
     client = Algorithmia.client()
     globals["SMID_ALGO"] = "algo://util/SmartImageDownloader/0.2.x"
-    globals["model"] = load_model(manifest["squeezenet"], client)
-    globals["labels"] = load_labels(manifest["label_file"], client)
+    globals["model"] = load_model(manifest["model_squeezenet"], client)
+    globals["labels"] = load_labels(manifest["labels"], client)
     return globals