test_luci_eval Namespace Reference

Functions
	luci_eval_verify (test_name, tflite_dir, circle_dir, eval_driver, rtolf32=1e-5, atolf32=1e-5)
	test_luci_eval (str test_name, str tflite_dir, str circle_dir, str eval_driver_path)
	recover_fromfile (path, test_name, suffix)
	recover_inputs (path, test_name)
	recover_outputs (path, test_name)
	save_binary_inputs (path, test_name, num_inputs, input_shape, input_dtype, input_data)
	luci_eval_verify (test_name, binary_path, eval_driver, rtolf32=1e-5, atolf32=1e-5)
	test_luci_eval (str default_test_name, str binary_path, str eval_driver_path)
	test_luci_eval_tol (str tol_test_name, str binary_path, str eval_driver_path, str rtolf32, str atolf32)

Function Documentation

luci_eval_verify() [1/2]

test_luci_eval.luci_eval_verify	(		test_name,
			binary_path,
			eval_driver,
			rtolf32 = 1e-5,
			atolf32 = 1e-5
	)

Definition at line 78 of file test_luci_eval.py.

def luci_eval_verify(test_name, binary_path, eval_driver, rtolf32=1e-5, atolf32=1e-5):
    circle_model = os.path.join(binary_path, test_name + ".circle")
 
    num_inputs, input_shape, input_dtype, input_data = recover_inputs(
        binary_path, test_name)
    assert num_inputs > 0, "No valid reference input file"
    save_binary_inputs(binary_path, test_name, num_inputs, input_shape, input_dtype,
                       input_data)
 
    num_ouputs, output_shape, output_dtype, output_data = recover_outputs(
        binary_path, test_name)
    assert num_ouputs > 0, "No valid reference output file"
 
    # Execute luci interpreter.
    subprocess.run([
        eval_driver, circle_model,
        str(num_inputs), circle_model + ".input", circle_model + ".output"
    ],
                   check=True)
 
    # Compare the results.
    for idx in range(num_ouputs):
        luci_output_data = np.fromfile(circle_model + ".output" + str(idx),
                                       output_dtype[idx])
        luci_output_data = np.reshape(luci_output_data, output_shape[idx])
        ref_output_data = np.reshape(output_data[idx], output_shape[idx])
 
        show_vals_and_stop = False
        if output_dtype[idx] == "float32":
            if not np.allclose(
                    luci_output_data, ref_output_data, rtol=rtolf32, atol=atolf32):
                show_vals_and_stop = True
        else:
            assert False, "Unsupported data type: " + output_dtype[idx]
 
        if show_vals_and_stop:
            print("\nreference:\n", ref_output_data)
            print("luci:\n", luci_output_data)
            message = "Execution result of " + test_name + " does not match with reference"
            assert False, message
 
 
# arguments must be in sync with `conftest.py`

References recover_inputs(), recover_outputs(), and save_binary_inputs().

luci_eval_verify() [2/2]

test_luci_eval.luci_eval_verify	(		test_name,
			tflite_dir,
			circle_dir,
			eval_driver,
			rtolf32 = 1e-5,
			atolf32 = 1e-5
	)

Definition at line 7 of file test_luci_eval.py.

                     atolf32=1e-5):
    tflite_model = os.path.join(tflite_dir, test_name + ".tflite")
    circle_model = os.path.join(circle_dir, test_name + ".after.circle")
 
    # NOTE reuse f32 value as int value too
    rtolint = int(rtolf32)
    atolint = int(atolf32)
 
    # Build TFLite interpreter.
    interpreter = tf.lite.Interpreter(tflite_model)
    interpreter.allocate_tensors()
 
    # Read SignatureDef and get output tensor id orders for remapping
    full_signatures = interpreter._get_full_signature_list()
    full_signatures_outputs_remap = None
    if full_signatures != None:
        signature_serving_default = full_signatures.get('serving_default', None)
        if signature_serving_default != None:
            signature_outputs = signature_serving_default['outputs']
 
            full_signatures_outputs_remap = []
            for index, (key, value) in enumerate(signature_outputs.items()):
                full_signatures_outputs_remap.append(value)
 
    # Generate random input data.
    num_inputs = len(interpreter.get_input_details())
    for i in range(num_inputs):
        input_details = interpreter.get_input_details()[i]
        if input_details["dtype"] == np.float32:
            input_data = np.array(np.random.random_sample(input_details["shape"]),
                                  input_details["dtype"])
        elif input_details["dtype"] == np.uint8:
            input_data = np.array(np.random.randint(0, 256, size=input_details["shape"]),
                                  input_details["dtype"])
        elif input_details["dtype"] == np.int16:
            input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
                                  input_details["dtype"])
        elif input_details["dtype"] == np.int32:
            input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
                                  input_details["dtype"])
        elif input_details["dtype"] == np.int64:
            input_data = np.array(np.random.randint(0, 100, size=input_details["shape"]),
                                  input_details["dtype"])
        elif input_details["dtype"] == np.bool_:
            input_data = np.array(
                np.random.choice(a=[True, False], size=input_details["shape"]),
                input_details["dtype"])
        else:
            assert False, "Unsupported input dtype"
 
        interpreter.set_tensor(input_details["index"], input_data)
        input_data.tofile(circle_model + ".input" + str(i))
 
    # Do inference
    interpreter.invoke()
 
    # Execute luci interpreter.
    subprocess.run([
        eval_driver, circle_model,
        str(num_inputs), circle_model + ".input", circle_model + ".output"
    ],
                   check=True)
 
    # Compare the results.
    inpt_output_details = interpreter.get_output_details()
    for idx in range(len(inpt_output_details)):
        output_details = inpt_output_details[idx]
        output_data = np.fromfile(circle_model + ".output" + str(idx),
                                  output_details["dtype"])
        shape_file = open(circle_model + ".output" + str(idx) + ".shape", 'r')
        output_shape = [int(i) for i in shape_file.read().split(',')]
        luci_output_data = np.reshape(output_data, output_shape)
        output_tensor = output_details["index"]
        if full_signatures_outputs_remap != None:
            output_tensor = full_signatures_outputs_remap[idx]
        intp_output_data = interpreter.get_tensor(output_tensor)
        err_msg = "Execution result of " + tflite_model + " does not match with " + circle_model
        if output_details["dtype"] == np.uint8:
            assert np.allclose(luci_output_data,
                               intp_output_data,
                               rtol=rtolint,
                               atol=atolint), err_msg
        elif output_details["dtype"] == np.float32:
            assert np.allclose(luci_output_data,
                               intp_output_data,
                               rtol=rtolf32,
                               atol=atolf32), err_msg
        elif output_details["dtype"] == np.int64:
            assert np.allclose(luci_output_data,
                               intp_output_data,
                               rtol=rtolint,
                               atol=atolint), err_msg
        elif output_details["dtype"] == np.int32:
            assert np.allclose(luci_output_data,
                               intp_output_data,
                               rtol=rtolint,
                               atol=atolint), err_msg
        elif output_details["dtype"] == np.int16:
            assert np.allclose(luci_output_data,
                               intp_output_data,
                               rtol=rtolint,
                               atol=atolint), err_msg
        elif output_details["dtype"] == np.bool_:
            assert np.allclose(luci_output_data, intp_output_data, rtol=0,
                               atol=0), err_msg
        else:
            assert False, "Unsupported data type: " + output_details["dtype"]
 
 
# arguments must be in sync with `conftest.py`

Referenced by test_luci_eval(), test_luci_eval(), and test_luci_eval_tol().

recover_fromfile()

test_luci_eval.recover_fromfile	(		path,
			test_name,
			suffix
	)

Definition at line 8 of file test_luci_eval.py.

def recover_fromfile(path, test_name, suffix):
    # .ref file format
    # 1'st line is shape, i.e. "2,4"
    # 2'nd line is dtype, i.e. "float32"
    # 3'rd line is comma seperated values
    ref_filename = test_name + ".ref." + suffix
    ref_datapath = os.path.join(path, ref_filename)
 
    num_data = 0
    parse_shape = []
    parse_dtype = []
    parse_value = []
 
    while True:
        refnum_filepath = ref_datapath + str(num_data)
        if (not os.path.isfile(refnum_filepath)):
            break
        with open(refnum_filepath, "r") as ref_file:
            lines = ref_file.readlines()
            assert len(lines) >= 3, "Invalid file: " + ref_filename + str(num_data)
        print("load reference data from", test_name)
        shape = [int(i) for i in lines[0].split(",")]
        dtype = lines[1].strip("\r\n \t")
        if dtype == "float32":
            value = [float(i) for i in lines[2].split(",")]
        else:
            assert False, "Unsupported data type: " + dtype
 
        # validate shape and number of elements
        num_elements = 1
        for dim in shape:
            num_elements = num_elements * dim
        if num_elements != len(value):
            assert False, "Number of value elements do not match with shape"
 
        parse_shape.append(shape)
        parse_dtype.append(dtype)
        parse_value.append(value)
 
        num_data = num_data + 1
 
    return num_data, parse_shape, parse_dtype, parse_value
 
 

Referenced by recover_inputs(), and recover_outputs().

recover_inputs()

test_luci_eval.recover_inputs	(		path,
			test_name
	)

Definition at line 52 of file test_luci_eval.py.

def recover_inputs(path, test_name):
    return recover_fromfile(path, test_name, "input")
 
 

References recover_fromfile().

Referenced by luci_eval_verify().

recover_outputs()

test_luci_eval.recover_outputs	(		path,
			test_name
	)

Definition at line 56 of file test_luci_eval.py.

def recover_outputs(path, test_name):
    return recover_fromfile(path, test_name, "output")
 
 
# save reference data to input files for luci-eval-driver

References recover_fromfile().

Referenced by luci_eval_verify().

save_binary_inputs()

test_luci_eval.save_binary_inputs	(		path,
			test_name,
			num_inputs,
			input_shape,
			input_dtype,
			input_data
	)

Definition at line 61 of file test_luci_eval.py.

def save_binary_inputs(path, test_name, num_inputs, input_shape, input_dtype, input_data):
    circle_inputpath = os.path.join(path, test_name + ".circle.input")
    for index in range(0, num_inputs):
        # reference input value
        if input_dtype[index] == "float32":
            nps = np.asarray(input_data[index], dtype=np.float32)
            nps.tofile(circle_inputpath + str(index))
        else:
            assert False, "Unsupported data type: " + input_dtype[index]
        # reference input shape
        nps = np.asarray(input_shape[index], dtype=np.short)
        nps.tofile(circle_inputpath + str(index) + ".shape", sep=",")
        # reference input dtype
        with open(circle_inputpath + str(index) + ".dtype", "w") as dtype_file:
            dtype_file.write(input_dtype[index])
 
 

Referenced by luci_eval_verify().

test_luci_eval() [1/2]

test_luci_eval.test_luci_eval	(	str	default_test_name,
		str	binary_path,
		str	eval_driver_path
	)

Definition at line 121 of file test_luci_eval.py.

def test_luci_eval(default_test_name: str, binary_path: str, eval_driver_path: str):
    luci_eval_verify(default_test_name, binary_path, eval_driver_path)
 
 
# arguments must be in sync with `conftest.py`

References luci_eval_verify().

test_luci_eval() [2/2]

test_luci_eval.test_luci_eval	(	str	test_name,
		str	tflite_dir,
		str	circle_dir,
		str	eval_driver_path
	)

Definition at line 122 of file test_luci_eval.py.

                   eval_driver_path: str):
    luci_eval_verify(test_name, tflite_dir, circle_dir, eval_driver_path)

References luci_eval_verify().

test_luci_eval_tol()

test_luci_eval.test_luci_eval_tol	(	str	tol_test_name,
		str	binary_path,
		str	eval_driver_path,
		str	rtolf32,
		str	atolf32
	)

Definition at line 126 of file test_luci_eval.py.

                       rtolf32: str, atolf32: str):
    luci_eval_verify(tol_test_name, binary_path, eval_driver_path, float(rtolf32),
                     float(atolf32))

References luci_eval_verify().