dio::hdf5::HDF5Importer Class Reference

#include <HDF5Importer.h>

Public Member Functions
	HDF5Importer (const std::string &path)
void	importGroup (const std::string &group)
void	readTensor (int32_t data_idx, int32_t input_idx, loco::DataType *dtype, std::vector< loco::Dimension > *shape, void *buffer, size_t buffer_bytes) const
	Read tensor data from file and store it into buffer.
void	readTensor (int32_t data_idx, int32_t input_idx, void *buffer, size_t buffer_bytes) const
bool	isRawData () const
int32_t	numData () const
int32_t	numInputs (int32_t data_idx) const

Detailed Description

Definition at line 41 of file HDF5Importer.h.

Constructor & Destructor Documentation

HDF5Importer()

dio::hdf5::HDF5Importer::HDF5Importer ( const std::string &  path )

explicit

Definition at line 117 of file HDF5Importer.cpp.

{
  if (_file.isHdf5(path) == false)
    throw std::runtime_error("Given data file is not HDF5");
 
  _file = H5::H5File(path, H5F_ACC_RDONLY);
}

Member Function Documentation

importGroup()

void dio::hdf5::HDF5Importer::importGroup ( const std::string &  group )

inline

Note

importGroup has to be called before readTensor is called Otherwise, readTensor will throw an exception

Definition at line 51 of file HDF5Importer.h.

{ _group = _file.openGroup(group); }

Referenced by mpqsolver::core::H5FileDataProvider::H5FileDataProvider(), record_minmax::HDF5Iterator::HDF5Iterator(), and dalgona::Dalgona::runAnalysisWithH5Input().

isRawData()

bool dio::hdf5::HDF5Importer::isRawData ( ) const

inline

Definition at line 69 of file HDF5Importer.h.

{ return _group.attrExists("rawData"); }

Referenced by mpqsolver::core::H5FileDataProvider::H5FileDataProvider(), record_minmax::HDF5Iterator::HDF5Iterator(), and dalgona::Dalgona::runAnalysisWithH5Input().

numData()

int32_t dio::hdf5::HDF5Importer::numData ( ) const

inline

Definition at line 71 of file HDF5Importer.h.

{ return _group.getNumObjs(); }

Referenced by record_minmax::HDF5Iterator::HDF5Iterator(), mpqsolver::core::H5FileDataProvider::numSamples(), and dalgona::Dalgona::runAnalysisWithH5Input().

numInputs()

int32_t dio::hdf5::HDF5Importer::numInputs

(

int32_t

data_idx

)

const

Definition at line 125 of file HDF5Importer.cpp.

{
  auto records = _group.openGroup(std::to_string(record_idx));
  return records.getNumObjs();
}

Referenced by record_minmax::HDF5Iterator::next(), mpqsolver::core::H5FileDataProvider::numInputs(), and dalgona::Dalgona::runAnalysisWithH5Input().

readTensor() [1/2]

void dio::hdf5::HDF5Importer::readTensor	(	int32_t	data_idx,
		int32_t	input_idx,
		loco::DataType *	dtype,
		std::vector< loco::Dimension > *	shape,
		void *	buffer,
		size_t	buffer_bytes
	)		const

Read tensor data from file and store it into buffer.

A tensor in the file can be retrieved with (data_idx, input_idx)

Parameters

data_idx	: index of the data
input_idx	: index of the input
dtype	: pointer to write the tensor's data type
shape	: pointer to write the tensor's shape
buffer	: pointer to write the tensor's data
buffer_bytes	: byte size of the buffer

Referenced by mpqsolver::core::H5FileDataProvider::getSampleInput(), record_minmax::HDF5Iterator::next(), and dalgona::Dalgona::runAnalysisWithH5Input().

readTensor() [2/2]

void dio::hdf5::HDF5Importer::readTensor	(	int32_t	data_idx,
		int32_t	input_idx,
		void *	buffer,
		size_t	buffer_bytes
	)		const

Definition at line 131 of file HDF5Importer.cpp.

{
  auto record = _group.openGroup(std::to_string(record_idx));
  auto tensor = record.openDataSet(std::to_string(input_idx));
 
  if (tensor.getInMemDataSize() != buffer_bytes)
    throw std::runtime_error("Buffer size does not match with the size of tensor data");
 
  readTensorData(tensor, static_cast<uint8_t *>(buffer));
}

---

The documentation for this class was generated from the following files:

• compiler/dio-hdf5/include/dio_hdf5/HDF5Importer.h
• compiler/dio-hdf5/src/HDF5Importer.cpp