ONE - On-device Neural Engine
Loading...
Searching...
No Matches
caffe2_importer.cpp
Go to the documentation of this file.
1/*
2 * Copyright (c) 2018 Samsung Electronics Co., Ltd. All Rights Reserved
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17#include "caffe2_importer.h"
18#include "caffe2/proto/caffe2.pb.h"
19#include "caffe2_op_types.h"
20#include "caffe2_op_creator.h"
21#include "caffe2_proto_helper.h"
22
23#include "mir/ops/InputOp.h"
24#include "mir/ops/OutputOp.h"
25
26#include <google/protobuf/io/zero_copy_stream_impl.h>
27#include <google/protobuf/io/coded_stream.h>
28
29#include <fcntl.h>
30
31#include <cassert>
32#include <cerrno>
33#include <cstring>
34#include <memory>
35#include <stdexcept>
36#include <utility>
37#include <set>
38
39namespace
40{
41
42using namespace mir_caffe2;
43
44class Caffe2Importer
45{
46public:
47 explicit Caffe2Importer(std::string predict_net, std::string init_net,
48 const std::vector<std::vector<int>> &input_shapes);
49
51 std::unique_ptr<mir::Graph> importModel();
52
53 ~Caffe2Importer();
54
55private:
56 std::string _predictNet;
57 std::string _initNet;
58 std::unique_ptr<mir::Graph> _graph;
59 std::unique_ptr<caffe2::NetDef> _predict_net;
60 std::unique_ptr<caffe2::NetDef> _init_net;
61 std::unique_ptr<Caffe2OpCreator> _opCreator;
62 std::vector<mir::Shape> _inputShapes;
63
64 static const std::map<std::string, SupportedCaffe2OpType> _operatorTypes;
65
66 // Maps Caffe2 operator input names to corresponding MIR operation outputs.
67 std::unordered_map<std::string, mir::Operation::Output *> _blobNameToOutput;
68
69 void import();
70 std::unique_ptr<mir::Graph> createIR();
71
76 void collectUnsupportedOps();
77
81 void createMIRNodesFromOp(const ::caffe2::OperatorDef &op);
82
86 std::vector<mir::Operation::Output *> getInputMIROps(const ::caffe2::OperatorDef &op);
87
88 void setOutputForTensor(const std::string &tensor_name, Operation::Output *output);
89 mir::Operation::Output *getOutputForTensor(const std::string &name) const;
90
94 void setGraphOutputs();
95};
96
97using namespace ::caffe2;
98using mir::Shape;
99
100Caffe2Importer::Caffe2Importer(std::string predict_net, std::string init_net,
101 const std::vector<std::vector<int>> &input_shapes)
102 : _predictNet(std::move(predict_net)), _initNet(std::move(init_net))
103{
104 for (auto &shape : input_shapes)
105 _inputShapes.emplace_back(shape);
106
107 _graph = std::make_unique<mir::Graph>();
108 _opCreator = std::make_unique<Caffe2OpCreator>(_graph.get());
109}
110
111Caffe2Importer::~Caffe2Importer() = default;
112
113static void loadModelFile(const std::string &filename, caffe2::NetDef *net)
114{
115 GOOGLE_PROTOBUF_VERIFY_VERSION;
116
117 int file_handle = open(filename.c_str(), O_RDONLY);
118
119 if (file_handle == -1)
120 throw std::runtime_error("Couldn't open file \"" + filename + "\": " + std::strerror(errno) +
121 ".");
122
123 google::protobuf::io::FileInputStream file_stream(file_handle);
124 file_stream.SetCloseOnDelete(true);
125
126 google::protobuf::io::CodedInputStream coded_stream(&file_stream);
127 coded_stream.SetTotalBytesLimit(INT_MAX, INT_MAX);
128
129 if (!net->ParseFromCodedStream(&coded_stream))
130 throw std::runtime_error("Couldn't parse file \"" + filename + "\".");
131
132 // If the file has not been consumed entirely, assume that the file is in the wrong format.
133 if (!coded_stream.ConsumedEntireMessage())
134 throw std::runtime_error("File \"" + filename + "\" has not been consumed entirely.");
135}
136
137void Caffe2Importer::import()
138{
139 _predict_net = std::make_unique<NetDef>();
140 loadModelFile(_predictNet, _predict_net.get());
141
142 _init_net = std::make_unique<NetDef>();
143 loadModelFile(_initNet, _init_net.get());
144
145 collectUnsupportedOps();
146}
147
148std::unique_ptr<mir::Graph> Caffe2Importer::createIR()
149{
150 // Load initializers.
151 for (const auto &op : _init_net->op())
152 createMIRNodesFromOp(op);
153
154 // Create inputs. This has to be done after processing initializers, because they may contain
155 // fake inputs.
156 // TODO Caffe2 does not provide a way to detect model inputs and outputs. For now assume that:
157 // - there is exactly one input;
158 // - the input is for the first layer;
159 // - the input has 'float' element type.
160 const auto &input_name = _predict_net->op(0).input(0);
161 mir::TensorType input_type(mir::DataType::FLOAT32, _inputShapes[0]);
162 auto input = _graph->create<mir::ops::InputOp>(input_type)->getOutput(0);
163 setOutputForTensor(input_name, input);
164
165 for (const auto &op : _predict_net->op())
166 createMIRNodesFromOp(op);
167
168 setGraphOutputs();
169
170 return std::move(_graph);
171}
172
173std::unique_ptr<mir::Graph> Caffe2Importer::importModel()
174{
175 import();
176 return createIR();
177}
178
179void Caffe2Importer::collectUnsupportedOps()
180{
181 std::set<std::string> unsupportedOps;
182 for (const auto &op : _predict_net->op())
183 {
184 if (_operatorTypes.find(op.type()) == _operatorTypes.end())
185 unsupportedOps.insert(op.type());
186 }
187
188 if (!unsupportedOps.empty())
189 {
190 std::string exceptionMsg("Can't load model, unsupported operators:");
191 for (const auto &op : unsupportedOps)
192 exceptionMsg.append("\n * " + op);
193 throw std::runtime_error(exceptionMsg);
194 }
195}
196
197void Caffe2Importer::createMIRNodesFromOp(const OperatorDef &op)
198{
199 std::vector<mir::Operation::Output *> outputs;
200
201 auto inputs = getInputMIROps(op);
202
203 SupportedCaffe2OpType opType = _operatorTypes.at(op.type());
204 switch (opType)
205 {
206 case SupportedCaffe2OpType::constantFill:
207 case SupportedCaffe2OpType::givenTensorFill:
208 case SupportedCaffe2OpType::givenTensorInt64Fill:
209 outputs = _opCreator->convertConstant(inputs, op);
210 break;
211 case SupportedCaffe2OpType::add:
212 outputs = _opCreator->convertAdd(inputs, op);
213 break;
214 case SupportedCaffe2OpType::averagePool:
215 outputs = _opCreator->convertAveragePool(inputs, op);
216 break;
217 case SupportedCaffe2OpType::conv:
218 outputs = _opCreator->convertConv(inputs, op);
219 break;
220 case SupportedCaffe2OpType::concat:
221 outputs = _opCreator->convertConcat(inputs, op);
222 break;
223 case SupportedCaffe2OpType::dropout:
224 outputs = _opCreator->convertDropout(inputs, op);
225 break;
226 case SupportedCaffe2OpType::FC:
227 outputs = _opCreator->convertFC(inputs, op);
228 break;
229 case SupportedCaffe2OpType::maxPool:
230 outputs = _opCreator->convertMaxPool(inputs, op);
231 break;
232 case SupportedCaffe2OpType::mul:
233 outputs = _opCreator->convertMul(inputs, op);
234 break;
235 case SupportedCaffe2OpType::relu:
236 outputs = _opCreator->convertRelu(inputs);
237 break;
238 case SupportedCaffe2OpType::resizeNearest:
239 outputs = _opCreator->convertResizeNearest(inputs, op);
240 break;
241 case SupportedCaffe2OpType::sigmoid:
242 outputs = _opCreator->convertSigmoid(inputs);
243 break;
244 case SupportedCaffe2OpType::softmax:
245 outputs = _opCreator->convertSoftmax(inputs, op);
246 break;
247 case SupportedCaffe2OpType::spatialBN:
248 outputs = _opCreator->convertSpatialBN(inputs, op);
249 break;
250 case SupportedCaffe2OpType::sum:
251 outputs = _opCreator->convertSum(inputs);
252 break;
253 case SupportedCaffe2OpType::clip:
254 outputs = _opCreator->convertClip(inputs, op);
255 break;
256 case SupportedCaffe2OpType::reshape:
257 outputs = _opCreator->convertReshape(inputs, op);
258 break;
259 default:
260 assert(false && "All unsupported types should have been found before this pass.");
261 }
262
263 for (size_t i = 0; i < outputs.size(); ++i)
264 {
265 setOutputForTensor(op.output(i), outputs[i]);
266 }
267}
268
269std::vector<mir::Operation::Output *> Caffe2Importer::getInputMIROps(const OperatorDef &op)
270{
271 std::vector<mir::Operation::Output *> inputs;
272
273 for (const auto &input_name : op.input())
274 {
275 inputs.push_back(getOutputForTensor(input_name));
276 }
277
278 return inputs;
279}
280
281void Caffe2Importer::setOutputForTensor(const std::string &tensor_name, Operation::Output *output)
282{
283 auto it = _blobNameToOutput.find(tensor_name);
284 if (it != _blobNameToOutput.cend())
285 {
286 // caffe2 input blob name could be same as output blob name, and next line will overwrite
287 // '_blobNameToOpOutput' element, but in all networks that I saw it was not a problem
288 it->second->setName("");
289 }
290 output->setName(tensor_name);
291 _blobNameToOutput[tensor_name] = output;
292}
293
294mir::Operation::Output *Caffe2Importer::getOutputForTensor(const std::string &name) const
295{
296 return _blobNameToOutput.at(name);
297}
298
299void Caffe2Importer::setGraphOutputs()
300{
301 // Create outputs.
302 // TODO Caffe2 does not provide a way to detect model inputs and outputs. For now assume that:
303 // - there is exactly one output;
304 // - the output is from the last layer.
305 const auto &output_name = _predict_net->op().rbegin()->output(0);
306 auto output = getOutputForTensor(output_name);
307 _graph->create<mir::ops::OutputOp>(output);
308}
309
310const std::map<std::string, SupportedCaffe2OpType> Caffe2Importer::_operatorTypes = {
311 {"Add", SupportedCaffe2OpType::add},
312 {"AveragePool", SupportedCaffe2OpType::averagePool},
313 {"Conv", SupportedCaffe2OpType::conv},
314 {"Concat", SupportedCaffe2OpType::concat},
315 {"ConstantFill", SupportedCaffe2OpType::constantFill},
316 {"Dropout", SupportedCaffe2OpType::dropout},
317 {"FC", SupportedCaffe2OpType::FC},
318 {"GivenTensorFill", SupportedCaffe2OpType::givenTensorFill},
319 {"MaxPool", SupportedCaffe2OpType::maxPool},
320 {"Mul", SupportedCaffe2OpType::mul},
321 {"Relu", SupportedCaffe2OpType::relu},
322 {"ResizeNearest", SupportedCaffe2OpType::resizeNearest},
323 {"Sigmoid", SupportedCaffe2OpType::sigmoid},
324 {"Softmax", SupportedCaffe2OpType::softmax},
325 {"SpatialBN", SupportedCaffe2OpType::spatialBN},
326 {"Sum", SupportedCaffe2OpType::sum},
327 {"Clip", SupportedCaffe2OpType::clip},
328 {"Reshape", SupportedCaffe2OpType::reshape},
329 {"GivenTensorInt64Fill", SupportedCaffe2OpType::givenTensorInt64Fill},
330};
331} // namespace
332
333namespace mir_caffe2
334{
335
336std::unique_ptr<mir::Graph> loadModel(std::string predict_net, std::string init_net,
337 const std::vector<std::vector<int>> &input_shapes)
338{
339 Caffe2Importer importer(std::move(predict_net), std::move(init_net), input_shapes);
340 return importer.importModel();
341}
342
343} // namespace mir_caffe2
Represents an output of a node.
Definition Operation.h:60
const char * tensor_name(const circle::Tensor *tensor)
std::unique_ptr< mir::Graph > loadModel(std::string predict_net, std::string init_net, const std::vector< std::vector< int > > &input_shapes)