record_minmax Namespace Reference

Data Structures
struct	DataBuffer
class	DataSetIterator
class	DirectoryIterator
class	HDF5Iterator
class	ListFileIterator
class	MinMaxComputer
class	MinMaxMap
class	MinMaxObserver
struct	MinMaxVectors
class	MovingAvgComputer
class	PercentileComputer
class	RandomIterator
class	RecordMinMax

Typedefs
using	Buffer = std::vector< char >
using	Output = std::vector< Buffer >
using	WholeOutput = std::vector< Output >

Functions
std::unique_ptr< MinMaxComputer >	make_percentile_computer (float min_percentile, float max_percentile)
std::unique_ptr< MinMaxComputer >	make_moving_avg_computer (uint32_t batch_size, float moving_avg_const)
float	getNthPercentile (std::vector< float > &vector, float percentile)
	getNthPercentile calculates the n-th percentile of input vector (0.0 <= n <= 100.0) linear interpolation is used when the desired percentile lies between two data points
float	getMovingAverage (const std::vector< float > &vector, const float alpha, const uint8_t batch_size, bool is_min)
	getMovingAverage calculates the weighted moving average of input vector The initial value is the minimum (or maximum) value of the first batch of the vector
uint32_t	numElements (const luci::CircleNode *node)
size_t	getTensorSize (const luci::CircleNode *node)
void	readDataFromFile (const std::string &filename, std::vector< char > &data, size_t data_size)
void	checkInputDimension (const luci::CircleInput *input)

Typedef Documentation

Buffer

using record_minmax::Buffer = typedef std::vector<char>

Definition at line 33 of file RecordMinMax.h.

Output

using record_minmax::Output = typedef std::vector<Buffer>

Definition at line 34 of file RecordMinMax.h.

WholeOutput

using record_minmax::WholeOutput = typedef std::vector<Output>

Definition at line 35 of file RecordMinMax.h.

Function Documentation

checkInputDimension()

void record_minmax::checkInputDimension

(

const luci::CircleInput *

input

)

Definition at line 29 of file Utils.cpp.

{
  assert(input); // FIX_CALLER_UNLESS
 
  for (uint32_t i = 0; i < input->rank(); i++)
    if (!input->dim(i).known())
      throw std::runtime_error(input->name() + " has unknown dimension");
 
  if (numElements(input) == 0)
    throw std::runtime_error(input->name() + " is a zero-sized input");
}

References numElements().

Referenced by record_minmax::RecordMinMax::profileData().

getMovingAverage()

float record_minmax::getMovingAverage	(	const std::vector< float > &	vector,
		const float	alpha,
		const uint8_t	batch_size,
		bool	is_min
	)

getMovingAverage calculates the weighted moving average of input vector The initial value is the minimum (or maximum) value of the first batch of the vector

Definition at line 60 of file RecordFunction.cpp.

{
  assert(!vector.empty());
  assert(alpha >= 0.0 && alpha <= 1.0);
  assert(batch_size > 0);
 
  auto getBatchMinOrMax = [&](uint32_t start_index) {
    assert(start_index < vector.size());
 
    float res = is_min ? std::numeric_limits<float>::max() : std::numeric_limits<float>::lowest();
    for (uint32_t offset = 0; offset < batch_size; offset++)
    {
      uint32_t index = start_index + offset;
      if (index >= vector.size())
        break;
 
      if (is_min)
      {
        res = vector[index] < res ? vector[index] : res;
      }
      else
      {
        res = vector[index] > res ? vector[index] : res;
      }
    }
    return res;
  };
 
  float curr_avg = getBatchMinOrMax(0);
  for (uint32_t i = batch_size; i < vector.size(); i += batch_size)
  {
    curr_avg = curr_avg * alpha + getBatchMinOrMax(i) * (1.0 - alpha);
  }
  return curr_avg;
}

References offset().

Referenced by record_minmax::MovingAvgComputer::update_qparam().

getNthPercentile()

float record_minmax::getNthPercentile	(	std::vector< float > &	vector,
		float	percentile
	)

getNthPercentile calculates the n-th percentile of input vector (0.0 <= n <= 100.0) linear interpolation is used when the desired percentile lies between two data points

Definition at line 30 of file RecordFunction.cpp.

{
  if (percentile < 0 || percentile > 100)
    throw std::runtime_error("Percentile must be ranged from 0 to 100");
 
  if (vector.empty())
    throw std::runtime_error("Percentile must take a non-empty vector as an argument");
 
  if (vector.size() == 1)
    return vector[0];
 
  std::vector<float> copy;
  copy.assign(vector.begin(), vector.end());
  std::sort(copy.begin(), copy.end());
 
  if (percentile == 0.0)
    return copy.front();
 
  if (percentile == 100.0)
    return copy.back();
 
  int index = static_cast<int>(std::floor((copy.size() - 1) * percentile / 100.0));
 
  float percent_i = static_cast<float>(index) / static_cast<float>(copy.size() - 1);
  float fraction =
    (percentile / 100.0 - percent_i) / ((index + 1.0) / (copy.size() - 1.0) - percent_i);
  float res = copy[index] + fraction * (copy[index + 1] - copy[index]);
  return res;
}

Referenced by record_minmax::PercentileComputer::update_qparam().

getTensorSize()

size_t record_minmax::getTensorSize

(

const luci::CircleNode *

node

)

Definition at line 57 of file Utils.cpp.

{
  assert(node); // FIX_CALLER_UNLESS
 
  uint32_t elem_size = luci::size(node->dtype());
  return numElements(node) * elem_size;
}

References numElements(), and luci::size().

Referenced by record_minmax::DirectoryIterator::next(), record_minmax::HDF5Iterator::next(), record_minmax::ListFileIterator::next(), and record_minmax::RandomIterator::next().

make_moving_avg_computer()

std::unique_ptr< MinMaxComputer > record_minmax::make_moving_avg_computer	(	uint32_t	batch_size,
		float	moving_avg_const
	)

Definition at line 80 of file MinMaxComputer.cpp.

{
  return std::make_unique<MovingAvgComputer>(batch_size, moving_avg_const);
}

make_percentile_computer()

std::unique_ptr< MinMaxComputer > record_minmax::make_percentile_computer	(	float	min_percentile,
		float	max_percentile
	)

Definition at line 75 of file MinMaxComputer.cpp.

{
  return std::make_unique<PercentileComputer>(min_percentile, max_percentile);
}

numElements()

uint32_t record_minmax::numElements

(

const luci::CircleNode *

node

)

Definition at line 41 of file Utils.cpp.

{
  assert(node); // FIX_CALLER_UNLESS
 
  uint32_t num_elements = 1;
  for (uint32_t i = 0; i < node->rank(); i++)
  {
    if (not node->dim(i).known())
      throw std::runtime_error("Unknown dimension found in " + node->name());
 
    num_elements *= node->dim(i).value();
  }
 
  return num_elements;
}

References luci::CircleNode::name().

Referenced by checkInputDimension(), getTensorSize(), and record_minmax::RandomIterator::next().

readDataFromFile()

void record_minmax::readDataFromFile	(	const std::string &	filename,
		std::vector< char > &	data,
		size_t	data_size
	)

Definition at line 65 of file Utils.cpp.

{
  assert(data.size() == data_size); // FIX_CALLER_UNLESS
 
  std::ifstream fs(filename, std::ifstream::binary);
  if (fs.fail())
    throw std::runtime_error("Cannot open file \"" + filename + "\".\n");
  if (fs.read(data.data(), data_size).fail())
    throw std::runtime_error("Failed to read data from file \"" + filename + "\".\n");
  if (fs.peek() != EOF)
    throw std::runtime_error("Input tensor size mismatches with \"" + filename + "\".\n");
}

Referenced by record_minmax::DirectoryIterator::next(), and record_minmax::ListFileIterator::next().