record_minmax::ListFileIterator Class Reference

#include <ListFileIterator.h>

Collaboration diagram for record_minmax::ListFileIterator:

Public Member Functions
	ListFileIterator (const std::string &input_path, luci::Module *module)
bool	hasNext () const override
std::vector< DataBuffer >	next () override
bool	check_type_shape () const override
Public Member Functions inherited from record_minmax::DataSetIterator
virtual	~DataSetIterator ()=default

Detailed Description

Definition at line 32 of file ListFileIterator.h.

Constructor & Destructor Documentation

ListFileIterator()

record_minmax::ListFileIterator::ListFileIterator	(	const std::string &	input_path,
		luci::Module *	module
	)

Definition at line 66 of file ListFileIterator.cpp.

{
  std::ifstream input_file(input_path);
  if (input_file.fail())
    throw std::runtime_error("Cannot open file \"" + input_path + "\".\n");
 
  auto input_nodes = loco::input_nodes(module->graph());
  for (auto input_node : input_nodes)
  {
    const auto cnode = loco::must_cast<const luci::CircleInput *>(input_node);
    _input_nodes.emplace_back(cnode);
  }
 
  std::string record;
  while (getline(input_file, record))
  {
    _lines.emplace_back(record);
  }
 
  if (_lines.size() == 0)
    throw std::runtime_error("The input data file does not contain any record.");
}

References loco::input_nodes(), and loco::must_cast().

Member Function Documentation

check_type_shape()

bool record_minmax::ListFileIterator::check_type_shape ( ) const

overridevirtual

Implements record_minmax::DataSetIterator.

Definition at line 151 of file ListFileIterator.cpp.

{ return false; }

hasNext()

bool record_minmax::ListFileIterator::hasNext ( ) const

overridevirtual

Implements record_minmax::DataSetIterator.

Definition at line 89 of file ListFileIterator.cpp.

{ return _curr_idx < _lines.size(); }

next()

std::vector< DataBuffer > record_minmax::ListFileIterator::next ( )

overridevirtual

Implements record_minmax::DataSetIterator.

Definition at line 91 of file ListFileIterator.cpp.

{
  const auto line = _lines.at(_curr_idx++);
 
  const auto file_names = parse_line(line);
 
  std::vector<DataBuffer> res;
 
  // Space-separated input files are written in a single line
  // This is the recommended way to write the list file
  if (file_names.size() == _input_nodes.size())
  {
    for (uint32_t i = 0; i < file_names.size(); i++)
    {
      DataBuffer buf;
      {
        const auto file_name = file_names.at(i);
        const auto input_node = _input_nodes.at(i);
        const auto input_size = getTensorSize(input_node);
 
        buf.data.resize(input_size);
 
        readDataFromFile(file_name, buf.data, input_size);
      }
 
      res.emplace_back(buf);
    }
  }
  else
  {
    // Must have a single file in one line (inputs are concatenated)
    if (file_names.size() != 1)
      throw std::runtime_error(
        "Wrong number of inputs are given. Model has " + std::to_string(_input_nodes.size()) +
        " inputs, but list file gives " + std::to_string(file_names.size()) + " inputs.");
 
    // Read data from file to buffer
    // Assumption: For a multi-input model, the binary file should have inputs concatenated in the
    // same order with the input index.
    // NOTE This is a legacy way to support multiple inputs.
    DataBuffer buf;
    {
      // Get total input size
      uint32_t total_input_size = 0;
      for (auto input_node : _input_nodes)
      {
        total_input_size += getTensorSize(input_node);
      }
 
      buf.data.resize(total_input_size);
 
      readDataFromFile(file_names.at(0), buf.data, total_input_size);
    }
 
    res.emplace_back(buf);
  }
 
  return res;
}

References record_minmax::getTensorSize(), and record_minmax::readDataFromFile().

---

The documentation for this class was generated from the following files:

• compiler/record-minmax/include/ListFileIterator.h
• compiler/record-minmax/src/ListFileIterator.cpp