Skip to content

Commit bd99599

Browse files
zeryxzeryx
committed
refactored and renamed manifest terms, added some basic testing and OOP
1 parent 5a295f9 commit bd99599

File tree

9 files changed

+78
-70
lines changed

9 files changed

+78
-70
lines changed

‎adk/ADK.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import os
44
import sys
55
from adk.io import create_exception, format_data, format_response
6-
from adk.manifest import ManifestData
6+
from adk.manifest.manifest import Manifest
77

88

99
class ADK(object):
@@ -35,7 +35,7 @@ def __init__(self, apply_func, load_func=None, client=None, manifest_path="model
3535
self.is_local = not os.path.exists(self.FIFO_PATH)
3636
self.load_result = None
3737
self.loading_exception = None
38-
self.manifest = ManifestData(client, manifest_path)
38+
self.manifest = Manifest(client, manifest_path)
3939

4040
def load(self):
4141
try:

‎adk/manifest/__init__.py‎

Whitespace-only changes.

‎adk/manifest/classes.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
2+
class FileData(object):
3+
def __init__(self, md5_checksum, file_path):
4+
self.md5_checksum = md5_checksum
5+
self.file_path = file_path
6+

adk/manifest.py renamed to adk/manifest/manifest.py

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import os
22
import json
33
import hashlib
4+
from adk.manifest.classes import FileData
45

56

6-
class ManifestData(object):
7+
class Manifest(object):
78
def __init__(self, client, model_manifest_path):
89
self.manifest_lock_path = model_manifest_path
910
self.manifest_data = get_manifest(self.manifest_lock_path)
@@ -19,51 +20,48 @@ def available(self):
1920
def initialize(self):
2021
if self.client is None:
2122
raise Exception("Client was not defined, please define a Client when using Model Manifests.")
22-
for required_file in self.manifest_data['required_models']:
23+
for required_file in self.manifest_data['required_files']:
2324
name = required_file['name']
2425
if name in self.models:
2526
raise Exception("Duplicate 'name' detected. \n"
2627
+ name + " was found to be used by more than one data file, please rename.")
27-
self.models[name] = {}
2828
expected_hash = required_file['md5_checksum']
29-
with self.client.file(required_file['data_api_path']).getFile() as f:
29+
with self.client.file(required_file['source_uri']).getFile() as f:
3030
local_data_path = f.name
3131
real_hash = md5_for_file(local_data_path)
3232
if real_hash != expected_hash and required_file['fail_on_tamper']:
3333
raise Exception("Model File Mismatch for " + name +
3434
"\nexpected hash: " + expected_hash + "\nreal hash: " + real_hash)
3535
else:
36-
self.models[name]["md5_checksum"] = real_hash
37-
self.models[name]['model_path'] = local_data_path
36+
self.models[name] = FileData(real_hash, local_data_path)
3837

3938
def get_model(self, model_name):
4039
if model_name in self.models:
41-
return self.models[model_name]['model_path']
42-
elif len([optional for optional in self.manifest_data['optional_models'] if
40+
return self.models[model_name].file_path
41+
elif len([optional for optional in self.manifest_data['optional_files'] if
4342
optional['name'] == model_name]) > 0:
4443
self.find_optional_model(model_name)
45-
return self.models[model_name]['model_path']
44+
return self.models[model_name].file_path
4645
else:
4746
raise Exception("model name " + model_name + " not found in manifest")
4847

49-
def find_optional_model(self, model_name):
48+
def find_optional_model(self, file_name):
5049

51-
found_models = [optional for optional in self.manifest_data['optional_models'] if
52-
optional['name'] == model_name]
50+
found_models = [optional for optional in self.manifest_data['optional_files'] if
51+
optional['name'] == file_name]
5352
if len(found_models) == 0:
54-
raise Exception("model with name '" + model_name + "' not found in model manifest.")
53+
raise Exception("file with name '" + file_name + "' not found in model manifest.")
5554
model_info = found_models[0]
56-
self.models[model_name] = {}
55+
self.models[file_name] = {}
5756
expected_hash = model_info['md5_checksum']
58-
with self.client.file(model_info['data_api_path']).getFile() as f:
57+
with self.client.file(model_info['source_uri']).getFile() as f:
5958
local_data_path = f.name
6059
real_hash = md5_for_file(local_data_path)
6160
if real_hash != expected_hash and model_info['fail_on_tamper']:
62-
raise Exception("Model File Mismatch for " + model_name +
61+
raise Exception("Model File Mismatch for " + file_name +
6362
"\nexpected hash: " + expected_hash + "\nreal hash: " + real_hash)
6463
else:
65-
self.models[model_name]["md5_checksum"] = real_hash
66-
self.models[model_name]['model_path'] = local_data_path
64+
self.models[file_name] = FileData(real_hash, local_data_path)
6765

6866

6967
def get_manifest(manifest_path):

‎examples/pytorch_image_classification/model_manifest.json‎

Lines changed: 14 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,29 @@
11
{
2-
"algorithm_name": "pytorch_image_classification",
3-
"required_models": [
4-
{
5-
"name": "model_squeezenet",
6-
"data_api_path": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
2+
"required_files" : [
3+
{ "name": "squeezenet",
4+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
75
"fail_on_tamper": true,
86
"metadata": {
9-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
7+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
108
}
119
},
1210
{
1311
"name": "labels",
14-
"data_api_path": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
12+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
1513
"fail_on_tamper": true,
1614
"metadata": {
17-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
15+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
1816
}
1917
}
2018
],
21-
"optional_models": [
22-
{
23-
"name": "vgg",
24-
"data_api_path": "data://AlgorithmiaSE/image_cassification_demo/vgg16-397923af.pth",
25-
"fail_on_tamper": false,
26-
"metadata": {
27-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
19+
"optional_files": [
20+
{
21+
"name": "mobilenet",
22+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth",
23+
"fail_on_tamper": false,
24+
"metadata": {
25+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
26+
}
2827
}
29-
}
3028
]
3129
}

‎examples/pytorch_image_classification/model_manifest.json.lock‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,34 @@
11
{
2-
"algorithm_name": "pytorch_image_classification",
3-
"model_tamper_behavior": "strict",
4-
"lock_version": "f7aec94cde3ca6274c6556ebe58ad3c6",
5-
"timestamp": "1632753496"
6-
"required_models" : [
7-
{ "name": "labels",
8-
"data_api_path": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
9-
"md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
2+
"timestamp": "1632770803",
3+
"lock_checksum": "0c1cd66787a7368ef31b23d653e31bf7",
4+
"required_files" : [
5+
{ "name": "squeezenet",
6+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
7+
"md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266",
108
"fail_on_tamper": true,
119
"metadata": {
12-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
10+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
1311
}
14-
}
15-
],
16-
"optional_models": [
12+
},
1713
{
18-
"name": "optional_data",
19-
"data_api_path": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
14+
"name": "labels",
15+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
2016
"md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
2117
"fail_on_tamper": true,
2218
"metadata": {
23-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
19+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
2420
}
2521
}
22+
],
23+
"optional_files": [
24+
{
25+
"name": "mobilenet",
26+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth",
27+
"md5_checksum": "f20b50b44fdef367a225d41f747a0963",
28+
"fail_on_tamper": false,
29+
"metadata": {
30+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
31+
}
32+
}
2633
]
2734
}

‎examples/pytorch_image_classification/src/Algorithm.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ def load(manifest):
5555

5656
state = {}
5757
state["SMID_ALGO"] = "algo://util/SmartImageDownloader/0.2.x"
58-
state["model"] = load_model(manifest.get_model("model_squeezenet"))
58+
state["model"] = load_model(manifest.get_model("squeezenet"))
5959
state["labels"] = load_labels(manifest.get_model("labels"))
6060
return state
6161

‎tests/manifests/bad_model_manifest.json.lock‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
{
22
"algorithm_name": "test_algorithm",
33
"timestamp": "1632770803",
4-
"lock_checksum": "57dcf8ab156f0c86f2f275919dcbf090",
5-
"required_models" : [
4+
"lock_checksum": "36162a15980fa79975d2a747eb1bb842",
5+
"required_files" : [
66
{ "name": "squeezenet",
7-
"data_api_path": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
7+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
88
"md5_checksum": "f20b50b44fdef367a225d41f747a0963",
99
"fail_on_tamper": true,
1010
"metadata": {
11-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
11+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
1212
}
1313
},
1414
{
@@ -17,18 +17,18 @@
1717
"md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
1818
"fail_on_tamper": true,
1919
"metadata": {
20-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
20+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
2121
}
2222
}
2323
],
24-
"optional_models": [
24+
"optional_files": [
2525
{
2626
"name": "mobilenet",
27-
"data_api_path": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth",
27+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth",
2828
"md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
2929
"fail_on_tamper": false,
3030
"metadata": {
31-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
31+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
3232
}
3333
}
3434
]

‎tests/manifests/good_model_manifest.json.lock‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,33 @@
11
{
2-
"algorithm_name": "test_algorithm",
32
"timestamp": "1632770803",
4-
"lock_checksum": "fba73f61886f0921b47997057e853d36",
5-
"required_models" : [
3+
"lock_checksum": "0c1cd66787a7368ef31b23d653e31bf7",
4+
"required_files" : [
65
{ "name": "squeezenet",
7-
"data_api_path": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
6+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/squeezenet1_1-f364aa15.pth",
87
"md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266",
98
"fail_on_tamper": true,
109
"metadata": {
11-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
10+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
1211
}
1312
},
1413
{
1514
"name": "labels",
16-
"data_api_path": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
15+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/imagenet_class_index.json",
1716
"md5_checksum": "c2c37ea517e94d9795004a39431a14cb",
1817
"fail_on_tamper": true,
1918
"metadata": {
20-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
19+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
2120
}
2221
}
2322
],
24-
"optional_models": [
23+
"optional_files": [
2524
{
2625
"name": "mobilenet",
27-
"data_api_path": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth",
26+
"source_uri": "data://AlgorithmiaSE/image_cassification_demo/mobilenet_v2-b0353104.pth",
2827
"md5_checksum": "f20b50b44fdef367a225d41f747a0963",
2928
"fail_on_tamper": false,
3029
"metadata": {
31-
"origination_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
30+
"dataset_md5_checksum": "46a44d32d2c5c07f7f66324bef4c7266"
3231
}
3332
}
3433
]

0 commit comments

Comments
 (0)