ONE - On-device Neural Engine
Loading...
Searching...
No Matches
luci_eval_verifier.py
Go to the documentation of this file.
1#!/usr/bin/env python3
2import numpy as np
3import tensorflow as tf
4import subprocess
5import argparse
6import traceback
7
8#
9# This script compares the execution result of luci-interpreter with that of TFLite interpreter
10#
11# Basic usage:
12# eval_verifier.py --driver build/compiler/luci-eval-driver/luci_eval_driver
13# --model inception_v3
14parser = argparse.ArgumentParser()
15parser.add_argument('--driver', type=str, required=True)
16parser.add_argument('--model', type=str, required=True)
17parser.add_argument('--rtolf32', type=str, required=False)
18parser.add_argument('--atolf32', type=str, required=False)
19args = parser.parse_args()
20
21driver = args.driver
22tflite_model = args.model + ".tflite"
23circle_model = args.model + ".circle"
24
25rtolf32 = 1e-5
26atolf32 = 1e-5
27# NOTE reuse f32 value as int value too
28rtolint = 0
29atolint = 0
30try:
31 if args.rtolf32 != None:
32 rtolf32 = float(args.rtolf32)
33 rtolint = int(rtolf32)
34 if args.atolf32 != None:
35 atolf32 = float(args.atolf32)
36 atolint = int(atolf32)
37except ValueError:
38 print("rtolf32 or atolf32 is not a number")
39 quit(128)
40
41# Build TFLite interpreter.
42interpreter = tf.lite.Interpreter(tflite_model)
43interpreter.allocate_tensors()
44
45# Read SignatureDef and get output tensor id orders for remapping
46full_signatures = interpreter._get_full_signature_list()
47full_signatures_outputs_remap = None
48if full_signatures != None:
49 signature_serving_default = full_signatures.get('serving_default', None)
50 if signature_serving_default != None:
51 signature_outputs = signature_serving_default['outputs']
52
53 full_signatures_outputs_remap = []
54 for index, (key, value) in enumerate(signature_outputs.items()):
55 full_signatures_outputs_remap.append(value)
56
57# Generate random input data.
58num_inputs = len(interpreter.get_input_details())
59for i in range(num_inputs):
60 input_details = interpreter.get_input_details()[i]
61 if input_details["dtype"] == np.float32:
62 input_data = np.array(np.random.random_sample(input_details["shape"]),
63 input_details["dtype"])
64 input_dtype = "float32"
65 elif input_details["dtype"] == np.uint8:
66 input_data = np.array(np.random.randint(0, 256, size=input_details["shape"]),
67 input_details["dtype"])
68 input_dtype = "uint8"
69 elif input_details["dtype"] == np.int16:
70 input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
71 input_details["dtype"])
72 input_dtype = "int16"
73 elif input_details["dtype"] == np.int32:
74 input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
75 input_details["dtype"])
76 input_dtype = "int32"
77 elif input_details["dtype"] == np.int64:
78 input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
79 input_details["dtype"])
80 input_dtype = "int64"
81 elif input_details["dtype"] == np.bool_:
82 input_data = np.array(
83 np.random.choice(a=[True, False], size=input_details["shape"]),
84 input_details["dtype"])
85 input_dtype = "bool"
86 else:
87 raise SystemExit("Unsupported input dtype")
88
89 interpreter.set_tensor(input_details["index"], input_data)
90 input_data.tofile(circle_model + ".input" + str(i))
91 input_details["shape"].tofile(circle_model + ".input" + str(i) + ".shape", sep=',')
92 with open(circle_model + ".input" + str(i) + ".dtype", 'w') as dtype_file:
93 dtype_file.write(input_dtype)
94
95# Do inference
96interpreter.invoke()
97
98# Execute luci interpreter.
99subprocess.run([
100 driver, circle_model,
101 str(num_inputs), circle_model + ".input", circle_model + ".output"
102],
103 check=True)
104
105# Compare the results.
106inpt_output_details = interpreter.get_output_details()
107for idx in range(len(inpt_output_details)):
108 output_details = inpt_output_details[idx]
109 output_data = np.fromfile(circle_model + ".output" + str(idx),
110 output_details["dtype"])
111 shape_file = open(circle_model + ".output" + str(idx) + ".shape", 'r')
112 output_shape = [int(i) for i in shape_file.read().split(',')]
113 luci_output_data = np.reshape(output_data, output_shape)
114 output_tensor = output_details["index"]
115 if full_signatures_outputs_remap != None:
116 output_tensor = full_signatures_outputs_remap[idx]
117 intp_output_data = interpreter.get_tensor(output_tensor)
118 try:
119 if output_details["dtype"] == np.uint8:
120 if np.allclose(luci_output_data, intp_output_data, rtol=rtolint,
121 atol=atolint) == False:
122 print("intp_output_data", intp_output_data)
123 print("luci_output_data", luci_output_data)
124 raise SystemExit("Execution result of " + tflite_model +
125 " does not match with " + circle_model)
126 output_dtype = "uint8"
127 elif output_details["dtype"] == np.float32:
128 if np.allclose(luci_output_data, intp_output_data, rtol=rtolf32,
129 atol=atolf32) == False:
130 print("intp_output_data", intp_output_data)
131 print("luci_output_data", luci_output_data)
132 raise SystemExit("Execution result of " + tflite_model +
133 " does not match with " + circle_model)
134 output_dtype = "float32"
135 elif output_details["dtype"] == np.int64:
136 if np.allclose(luci_output_data, intp_output_data, rtol=rtolint,
137 atol=atolint) == False:
138 print("intp_output_data", intp_output_data)
139 print("luci_output_data", luci_output_data)
140 raise SystemExit("Execution result of " + tflite_model +
141 " does not match with " + circle_model)
142 output_dtype = "int64"
143 elif output_details["dtype"] == np.int32:
144 if np.allclose(luci_output_data, intp_output_data, rtol=rtolint,
145 atol=atolint) == False:
146 print("intp_output_data", intp_output_data)
147 print("luci_output_data", luci_output_data)
148 raise SystemExit("Execution result of " + tflite_model +
149 " does not match with " + circle_model)
150 output_dtype = "int32"
151 elif output_details["dtype"] == np.int16:
152 if np.allclose(luci_output_data, intp_output_data, rtol=rtolint,
153 atol=atolint) == False:
154 print("intp_output_data", intp_output_data)
155 print("luci_output_data", luci_output_data)
156 raise SystemExit("Execution result of " + tflite_model +
157 " does not match with " + circle_model)
158 output_dtype = "int16"
159 elif output_details["dtype"] == np.bool_:
160 if np.allclose(luci_output_data, intp_output_data, rtol=0, atol=0) == False:
161 print("intp_output_data", intp_output_data)
162 print("luci_output_data", luci_output_data)
163 raise SystemExit("Execution result of " + tflite_model +
164 " does not match with " + circle_model)
165 output_dtype = "bool"
166 else:
167 raise SystemExit("Unsupported data type: ", output_details["dtype"])
168
169 # save outputN.dtype file
170 with open(circle_model + ".output" + str(idx) + ".dtype", 'w') as dtype_file:
171 dtype_file.write(output_dtype)
172 except:
173 print(traceback.format_exc())
174 quit(255)
175
176quit(0)