Skip to content

Commit

Permalink
Change CMakeLists: added new thirdparty libraries
Browse files Browse the repository at this point in the history
  • Loading branch information
Romanov committed May 19, 2021
1 parent 09b3a2f commit 5fd7058
Show file tree
Hide file tree
Showing 9 changed files with 118 additions and 185 deletions.
23 changes: 23 additions & 0 deletions inference-engine/samples/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,30 @@ if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/gflags")
add_subdirectory(thirdparty/gflags EXCLUDE_FROM_ALL)
set_target_properties(gflags_nothreads_static PROPERTIES FOLDER thirdparty)
endfunction()

function(add_zlib)
if(NOT WIN32)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-all")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-all")
endif()
set(BUILD_SHARED_LIBS OFF)
add_subdirectory(thirdparty/zlib EXCLUDE_FROM_ALL)
set_target_properties(zlib PROPERTIES FOLDER thirdparty)
endfunction()

function(add_cnpy)
if(NOT WIN32)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-all")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-all")
endif()
set(BUILD_SHARED_LIBS OFF)
add_subdirectory(thirdparty/cnpy EXCLUDE_FROM_ALL)
set_target_properties(cnpy PROPERTIES FOLDER thirdparty)
endfunction()

add_gflags()
add_zlib()
add_cnpy()
endif()

if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
Expand Down
4 changes: 0 additions & 4 deletions inference-engine/samples/speech_sample/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,6 @@
# SPDX-License-Identifier: Apache-2.0
#

if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/cnpy")
add_subdirectory(cnpy)
endif()

file (GLOB SRC ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp
file (GLOB HDR ${CMAKE_CURRENT_SOURCE_DIR}/*.hpp)
${CMAKE_CURRENT_SOURCE_DIR}/*.h)
Expand Down
104 changes: 38 additions & 66 deletions inference-engine/samples/speech_sample/fileutils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,9 @@
// SPDX-License-Identifier: Apache-2.0
//

#include <string>
#include <iostream>

#include <samples/common.hpp>
#include <samples/slog.hpp>

#include "fileutils.hpp"
#include "cnpy/cnpy.h"


void ArkFile::GetFileInfo(const char* fileName,
uint32_t numArrayToFindSize,
uint32_t* ptrNumArrays,
uint32_t* ptrNumMemoryBytes) {
void ArkFile::GetFileInfo(const char* fileName, uint32_t numArrayToFindSize, uint32_t* ptrNumArrays, uint32_t* ptrNumMemoryBytes) {
uint32_t numArrays = 0;
uint32_t numMemoryBytes = 0;

Expand All @@ -30,30 +19,30 @@ void ArkFile::GetFileInfo(const char* fileName,
break;
}
in_file.read(reinterpret_cast<char*>(&numRows), sizeof(uint32_t)); // read number of rows
std::getline(in_file, line, '\4'); // read control-D
std::getline(in_file, line, '\4'); // read control-D
in_file.read(reinterpret_cast<char*>(&numCols), sizeof(uint32_t)); // read number of columns
num_bytes = numRows * numCols * sizeof(float);
in_file.seekg(num_bytes, in_file.cur); // read data
in_file.seekg(num_bytes, in_file.cur); // read data

if (numArrays == numArrayToFindSize) {
numMemoryBytes += num_bytes;
}
numArrays++;
}
in_file.close();
}
else {
} else {
fprintf(stderr, "Failed to open %s for reading in GetKaldiArkInfo()!\n", fileName);
exit(-1);
}

if (ptrNumArrays != NULL) *ptrNumArrays = numArrays;
if (ptrNumMemoryBytes != NULL) *ptrNumMemoryBytes = numMemoryBytes;
if (ptrNumArrays != NULL)
*ptrNumArrays = numArrays;
if (ptrNumMemoryBytes != NULL)
*ptrNumMemoryBytes = numMemoryBytes;
}


void ArkFile::LoadFile(const char* fileName, uint32_t arrayIndex, std::string& ptrName, std::vector<uint8_t>& memory,
uint32_t* ptrNumRows, uint32_t* ptrNumColumns, uint32_t* ptrNumBytesPerElement) {
void ArkFile::LoadFile(const char* fileName, uint32_t arrayIndex, std::string& ptrName, std::vector<uint8_t>& memory, uint32_t* ptrNumRows,
uint32_t* ptrNumColumns, uint32_t* ptrNumBytesPerElement) {
std::ifstream in_file(fileName, std::ios::binary);
if (in_file.good()) {
uint32_t i = 0;
Expand All @@ -65,43 +54,36 @@ void ArkFile::LoadFile(const char* fileName, uint32_t arrayIndex, std::string& p
if (line.compare("BFM ") != 0) {
break;
}
in_file.read(reinterpret_cast<char*>(&numRows), sizeof(uint32_t)); // read number of rows
std::getline(in_file, line, '\4'); // read control-D
in_file.read(reinterpret_cast<char*>(&numCols), sizeof(uint32_t)); // read number of columns
in_file.seekg(numRows * numCols * sizeof(float), in_file.cur); // read data
in_file.read(reinterpret_cast<char*>(&numRows), sizeof(uint32_t)); // read number of rows
std::getline(in_file, line, '\4'); // read control-D
in_file.read(reinterpret_cast<char*>(&numCols), sizeof(uint32_t)); // read number of columns
in_file.seekg(numRows * numCols * sizeof(float), in_file.cur); // read data
i++;
}
if (!in_file.eof()) {
std::string line;
std::getline(in_file, ptrName, '\0'); // read variable length name followed by space and NUL
std::getline(in_file, line, '\4'); // read "BFM" followed by space and control-D
std::getline(in_file, ptrName, '\0'); // read variable length name followed by space and NUL
std::getline(in_file, line, '\4'); // read "BFM" followed by space and control-D
if (line.compare("BFM ") != 0) {
fprintf(stderr, "Cannot find array specifier in file %s in LoadKaldiArkArray()!\n", fileName);
exit(-1);
}
in_file.read(reinterpret_cast<char*>(ptrNumRows), sizeof(uint32_t)); // read number of rows
std::getline(in_file, line, '\4'); // read control-D
in_file.read(reinterpret_cast<char*>(ptrNumColumns), sizeof(uint32_t)); // read number of columns
in_file.read(reinterpret_cast<char*>(ptrNumRows), sizeof(uint32_t)); // read number of rows
std::getline(in_file, line, '\4'); // read control-D
in_file.read(reinterpret_cast<char*>(ptrNumColumns), sizeof(uint32_t)); // read number of columns
in_file.read(reinterpret_cast<char*>(&memory.front()),
*ptrNumRows * *ptrNumColumns * sizeof(float)); // read array data
*ptrNumRows * *ptrNumColumns * sizeof(float)); // read array data
}
in_file.close();
}
else {
} else {
fprintf(stderr, "Failed to open %s for reading in GetKaldiArkInfo()!\n", fileName);
exit(-1);
}

*ptrNumBytesPerElement = sizeof(float);
}


void ArkFile::SaveFile(const char* fileName,
bool shouldAppend,
std::string name,
void* ptrMemory,
uint32_t numRows,
uint32_t numColumns) {
void ArkFile::SaveFile(const char* fileName, bool shouldAppend, std::string name, void* ptrMemory, uint32_t numRows, uint32_t numColumns) {
std::ios_base::openmode mode = std::ios::binary;
if (shouldAppend) {
mode |= std::ios::app;
Expand All @@ -117,17 +99,12 @@ void ArkFile::SaveFile(const char* fileName,
out_file.write(reinterpret_cast<char*>(&numColumns), sizeof(uint32_t));
out_file.write(reinterpret_cast<char*>(ptrMemory), numRows * numColumns * sizeof(float));
out_file.close();
}
else {
} else {
throw std::runtime_error(std::string("Failed to open %s for writing in SaveKaldiArkArray()!\n") + fileName);
}
}


void NumpyFile::GetFileInfo(const char* fileName,
uint32_t numArrayToFindSize,
uint32_t* ptrNumArrays,
uint32_t* ptrNumMemoryBytes) {
void NumpyFile::GetFileInfo(const char* fileName, uint32_t numArrayToFindSize, uint32_t* ptrNumArrays, uint32_t* ptrNumMemoryBytes) {
uint32_t numArrays = 0;
uint32_t numMemoryBytes = 0;

Expand All @@ -139,37 +116,32 @@ void NumpyFile::GetFileInfo(const char* fileName,
cnpy::NpyArray my_npy = it->second;
numMemoryBytes = my_npy.data_holder->size();

if (ptrNumArrays != NULL) *ptrNumArrays = numArrays;
if (ptrNumMemoryBytes != NULL) *ptrNumMemoryBytes = numMemoryBytes;
if (ptrNumArrays != NULL)
*ptrNumArrays = numArrays;
if (ptrNumMemoryBytes != NULL)
*ptrNumMemoryBytes = numMemoryBytes;
}


void NumpyFile::LoadFile(const char* fileName, uint32_t arrayIndex, std::string& ptrName, std::vector<uint8_t>& memory,
uint32_t* ptrNumRows, uint32_t* ptrNumColumns, uint32_t* ptrNumBytesPerElement) {
void NumpyFile::LoadFile(const char* fileName, uint32_t arrayIndex, std::string& ptrName, std::vector<uint8_t>& memory, uint32_t* ptrNumRows,
uint32_t* ptrNumColumns, uint32_t* ptrNumBytesPerElement) {
cnpy::npz_t my_npz1 = cnpy::npz_load(fileName);
auto it = my_npz1.begin();
std::advance(it, arrayIndex);
ptrName = it->first;
cnpy::NpyArray my_npy = it->second;
*ptrNumRows = my_npy.shape[0];
*ptrNumColumns = my_npy.shape[1];
*ptrNumRows = my_npy.shape[0];
*ptrNumColumns = my_npy.shape[1];

for (size_t i = 0; i < my_npy.data_holder->size(); i++) {
memory.at(i) = my_npy.data_holder->at(i);
}

for (int i = 0; i < my_npy.data_holder->size(); i++) {
memory.at(i) = my_npy.data_holder->at(i);
}

*ptrNumBytesPerElement = sizeof(float);
}


void NumpyFile::SaveFile(const char* fileName,
bool shouldAppend,
std::string name,
void* ptrMemory,
uint32_t numRows,
uint32_t numColumns) {
void NumpyFile::SaveFile(const char* fileName, bool shouldAppend, std::string name, void* ptrMemory, uint32_t numRows, uint32_t numColumns) {
std::string mode;
shouldAppend ? mode = "a" : mode = "w";
std::vector<size_t> shape{ numRows , numColumns };
std::vector<size_t> shape {numRows, numColumns};
cnpy::npz_save(fileName, name, reinterpret_cast<float*>(ptrMemory), shape, mode);
}
87 changes: 19 additions & 68 deletions inference-engine/samples/speech_sample/fileutils.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,52 +3,34 @@
//

#pragma once
#include <cnpy/cnpy.h>

class BaseFile {
#include <samples/common.hpp>
#include <samples/slog.hpp>

class BaseFile {
public:
virtual void LoadFile(const char* fileName,
uint32_t arrayIndex,
std::string& ptrName,
std::vector<uint8_t>& memory,
uint32_t* ptrNumRows,
uint32_t* ptrNumColumns,
uint32_t* ptrNumBytesPerElement)= 0;

virtual void SaveFile(const char* fileName,
bool shouldAppend,
std::string name,
void* ptrMemory,
uint32_t numRows,
uint32_t numColumns)= 0;

virtual void LoadFile(const char* fileName, uint32_t arrayIndex, std::string& ptrName, std::vector<uint8_t>& memory, uint32_t* ptrNumRows,
uint32_t* ptrNumColumns, uint32_t* ptrNumBytesPerElement) = 0;

virtual void GetFileInfo(const char* fileName,
uint32_t numArrayToFindSize,
uint32_t* ptrNumArrays,
uint32_t* ptrNumMemoryBytes)= 0;
virtual void SaveFile(const char* fileName, bool shouldAppend, std::string name, void* ptrMemory, uint32_t numRows, uint32_t numColumns) = 0;

virtual void GetFileInfo(const char* fileName, uint32_t numArrayToFindSize, uint32_t* ptrNumArrays, uint32_t* ptrNumMemoryBytes) = 0;
};

class ArkFile : public BaseFile {

public:
/**
/**
* @brief Get info from Kaldi ARK speech feature vector file
* @param fileName .ark file name
* @param numArrayToFindSize number speech feature vectors in the file
* @param ptrNumArrays pointer to specific number array
* @param ptrNumMemoryBytes pointer to specific number of memory bytes
* @return none.
*/
virtual void GetFileInfo(const char* fileName,
uint32_t numArrayToFindSize,
uint32_t* ptrNumArrays,
uint32_t* ptrNumMemoryBytes);


virtual void GetFileInfo(const char* fileName, uint32_t numArrayToFindSize, uint32_t* ptrNumArrays, uint32_t* ptrNumMemoryBytes);

/**
/**
* @brief Load Kaldi ARK speech feature vector file
* @param fileName .ark file name
* @param arrayIndex number speech feature vector in the file
Expand All @@ -59,16 +41,10 @@ class ArkFile : public BaseFile {
* @param ptrNumBytesPerElement pointer to number bytes per element (size of float by default)
* @return none.
*/
void LoadFile(const char* fileName,
uint32_t arrayIndex,
std::string& ptrName,
std::vector<uint8_t>& memory,
uint32_t* ptrNumRows,
uint32_t* ptrNumColumns,
uint32_t* ptrNumBytesPerElement);
virtual void LoadFile(const char* fileName, uint32_t arrayIndex, std::string& ptrName, std::vector<uint8_t>& memory, uint32_t* ptrNumRows,
uint32_t* ptrNumColumns, uint32_t* ptrNumBytesPerElement);


/**
/**
* @brief Save Kaldi ARK speech feature vector file
* @param fileName .ark file name
* @param shouldAppend bool flag to rewrite or add to the end of file
Expand All @@ -78,40 +54,15 @@ class ArkFile : public BaseFile {
* @param numColumns number of columns
* @return none.
*/
void SaveFile(const char* fileName,
bool shouldAppend,
std::string name,
void* ptrMemory,
uint32_t numRows,
uint32_t numColumns);

virtual void SaveFile(const char* fileName, bool shouldAppend, std::string name, void* ptrMemory, uint32_t numRows, uint32_t numColumns);
};


class NumpyFile : public BaseFile {

public:

void GetFileInfo(const char* fileName,
uint32_t numArrayToFindSize,
uint32_t* ptrNumArrays,
uint32_t* ptrNumMemoryBytes);


void LoadFile(const char* fileName,
uint32_t arrayIndex,
std::string& ptrName,
std::vector<uint8_t>& memory,
uint32_t* ptrNumRows,
uint32_t* ptrNumColumns,
uint32_t* ptrNumBytesPerElement);

virtual void GetFileInfo(const char* fileName, uint32_t numArrayToFindSize, uint32_t* ptrNumArrays, uint32_t* ptrNumMemoryBytes);

void SaveFile(const char* fileName,
bool shouldAppend,
std::string name,
void* ptrMemory,
uint32_t numRows,
uint32_t numColumns);
virtual void LoadFile(const char* fileName, uint32_t arrayIndex, std::string& ptrName, std::vector<uint8_t>& memory, uint32_t* ptrNumRows,
uint32_t* ptrNumColumns, uint32_t* ptrNumBytesPerElement);

virtual void SaveFile(const char* fileName, bool shouldAppend, std::string name, void* ptrMemory, uint32_t numRows, uint32_t numColumns);
};
Loading

0 comments on commit 5fd7058

Please sign in to comment.