31 def collect_data_path(self, fp32_dir, fq_dir):
32
33
34
35
36
37
38
39
40
41
42
43
44
45 self._num_data = len(list(filter(os.path.isdir, glob.glob(fp32_dir + '/*'))))
46 if self._num_data != len(list(filter(os.path.isdir, glob.glob(fq_dir + '/*')))):
47 raise RuntimeError("Number of data mistmatches")
48
49 self._num_processed_data += self._num_data
50
51 self._tid_to_tname = dict()
52 with open(Path(fp32_dir) / 'tensors.json') as f:
53 tname_to_tid = json.load(f)
54
55 for tname, tid in tname_to_tid.items():
56 self._tid_to_tname[tid] = tname
57
58
59
60
61
62
63
64
65 data_paths = dict()
66 for data_idx in range(self._num_data):
67 fp32_results = glob.glob(fp32_dir + '/' + str(data_idx) + '/*.npy')
68 for fp32_data_path in fp32_results:
69 fp32_path = Path(fp32_data_path)
70 fq_data_path = fq_dir + '/' + str(data_idx) + '/' + fp32_path.with_suffix(
71 '.npy').name
72 fq_path = Path(fq_data_path)
73 tid = int(fp32_path.stem)
74 tensor_name = self._tid_to_tname[tid]
75
76
77 if fq_path.is_file() and fp32_path.is_file():
78 if tensor_name in data_paths:
79 data_paths[tensor_name].append((fp32_data_path, fq_data_path))
80 else:
81 data_paths[tensor_name] = [(fp32_data_path, fq_data_path)]
82
83 return data_paths
84
1.9.8