Skip to content

Commit

Permalink
Changed cnpy library: some methods and fixed Klocwork issues
Browse files Browse the repository at this point in the history
  • Loading branch information
Romanov committed May 31, 2021
1 parent d03fe4a commit 1f6255e
Show file tree
Hide file tree
Showing 7 changed files with 72 additions and 69 deletions.
1 change: 1 addition & 0 deletions .gitmodules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,4 @@
[submodule "thirdparty/zlib/zlib"]
path = thirdparty/zlib/zlib
url = https://github.com/madler/zlib.git
ignore = dirty
1 change: 1 addition & 0 deletions thirdparty/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@ add_subdirectory(ittapi)
add_subdirectory(itt_collector)
add_subdirectory(xbyak EXCLUDE_FROM_ALL)
add_subdirectory(zlib EXCLUDE_FROM_ALL)
add_subdirectory(cnpy EXCLUDE_FROM_ALL)
openvino_developer_export_targets(COMPONENT openvino_common TARGETS xbyak)
31 changes: 12 additions & 19 deletions thirdparty/cnpy/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,19 @@ endif(COMMAND cmake_policy)

project(CNPY)

set(BUILD_SHARED_LIBS OFF)
set(TARGET_NAME "cnpy")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
add_library(cnpy STATIC "cnpy.cpp")

option(ENABLE_STATIC "Build static (.a) library" ON)
if(NOT WIN32)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-all")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-all")
target_compile_options(${TARGET_NAME} PUBLIC -Wno-unused-variable)
endif()

find_package(ZLIB REQUIRED)
target_link_libraries(${TARGET_NAME} PUBLIC zlib)
target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}"
"${CMAKE_CURRENT_SOURCE_DIR}/..")

include_directories(${ZLIB_INCLUDE_DIRS})

add_library(cnpy SHARED "cnpy.cpp")
target_link_libraries(cnpy ${ZLIB_LIBRARIES})
install(TARGETS "cnpy" LIBRARY DESTINATION lib PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)

if(ENABLE_STATIC)
add_library(cnpy-static STATIC "cnpy.cpp")
set_target_properties(cnpy-static PROPERTIES OUTPUT_NAME "cnpy")
install(TARGETS "cnpy-static" ARCHIVE DESTINATION lib)
endif(ENABLE_STATIC)

install(FILES "cnpy.h" DESTINATION include)
install(FILES "mat2npz" "npy2mat" "npz2mat" DESTINATION bin PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)

add_executable(example1 example1.cpp)
target_link_libraries(example1 cnpy)
set_target_properties(cnpy PROPERTIES FOLDER thirdparty)
72 changes: 47 additions & 25 deletions thirdparty/cnpy/cnpy.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,17 @@ void cnpy::parse_npy_header(unsigned char* buffer,size_t& word_size, std::vector

void cnpy::parse_npy_header(FILE* fp, size_t& word_size, std::vector<size_t>& shape, bool& fortran_order) {
char buffer[256];
size_t res = fread(buffer,sizeof(char),11,fp);
if(res != 11)
std::string header;
size_t res = fread(buffer,sizeof(char),11,fp);
if (res != 11)
throw std::runtime_error("parse_npy_header: failed fread");
std::string header = fgets(buffer,256,fp);
char * data = fgets(buffer, 256, fp);
if (data != NULL) {
header = data;
}
else {
header = "";
}
assert(header[header.size()-1] == '\n');

size_t loc1, loc2;
Expand Down Expand Up @@ -180,12 +187,16 @@ cnpy::NpyArray load_the_npy_file(FILE* fp) {
size_t word_size;
bool fortran_order;
cnpy::parse_npy_header(fp,word_size,shape,fortran_order);

cnpy::NpyArray arr(shape, word_size, fortran_order);
size_t nread = fread(arr.data<char>(),1,arr.num_bytes(),fp);
if(nread != arr.num_bytes())
throw std::runtime_error("load_the_npy_file: failed fread");
return arr;
if (word_size >= 0 && word_size < ULLONG_MAX) {
cnpy::NpyArray arr(shape, word_size, fortran_order);
size_t nread = fread(arr.data<char>(), 1, arr.num_bytes(), fp);
if (nread != arr.num_bytes())
throw std::runtime_error("load_the_npy_file: failed fread");
return arr;
}
else {
throw std::runtime_error("load_the_npy_file: incorrect word_size");
}
}

cnpy::NpyArray load_the_npz_array(FILE* fp, uint32_t compr_bytes, uint32_t uncompr_bytes) {
Expand Down Expand Up @@ -218,13 +229,17 @@ cnpy::NpyArray load_the_npz_array(FILE* fp, uint32_t compr_bytes, uint32_t uncom
size_t word_size;
bool fortran_order;
cnpy::parse_npy_header(&buffer_uncompr[0],word_size,shape,fortran_order);
if (word_size >= 0 && word_size < ULLONG_MAX) {
cnpy::NpyArray array(shape, word_size, fortran_order);

cnpy::NpyArray array(shape, word_size, fortran_order);

size_t offset = uncompr_bytes - array.num_bytes();
memcpy(array.data<unsigned char>(),&buffer_uncompr[0]+offset,array.num_bytes());
size_t offset = uncompr_bytes - array.num_bytes();
memcpy(array.data<unsigned char>(), &buffer_uncompr[0] + offset, array.num_bytes());

return array;
return array;
}
else {
throw std::runtime_error("load_the_npz_array: incorrect word_size");
}
}

cnpy::npz_t cnpy::npz_load(std::string fname) {
Expand All @@ -239,19 +254,21 @@ cnpy::npz_t cnpy::npz_load(std::string fname) {
while(1) {
std::vector<char> local_header(30);
size_t headerres = fread(&local_header[0],sizeof(char),30,fp);
if(headerres != 30)
if (headerres != 30) {
fclose(fp);
throw std::runtime_error("npz_load: failed fread");

}
//if we've reached the global header, stop reading
if(local_header[2] != 0x03 || local_header[3] != 0x04) break;

//read in the variable name
uint16_t name_len = *(uint16_t*) &local_header[26];
std::string varname(name_len,' ');
size_t vname_res = fread(&varname[0],sizeof(char),name_len,fp);
if(vname_res != name_len)
if (vname_res != name_len) {
fclose(fp);
throw std::runtime_error("npz_load: failed fread");

}
//erase the lagging .npy
varname.erase(varname.end()-4,varname.end());

Expand All @@ -260,16 +277,18 @@ cnpy::npz_t cnpy::npz_load(std::string fname) {
if(extra_field_len > 0) {
std::vector<char> buff(extra_field_len);
size_t efield_res = fread(&buff[0],sizeof(char),extra_field_len,fp);
if(efield_res != extra_field_len)
if (efield_res != extra_field_len) {
fclose(fp);
throw std::runtime_error("npz_load: failed fread");
}
}

uint16_t compr_method = *reinterpret_cast<uint16_t*>(&local_header[0]+8);
uint32_t compr_bytes = *reinterpret_cast<uint32_t*>(&local_header[0]+18);
uint32_t uncompr_bytes = *reinterpret_cast<uint32_t*>(&local_header[0]+22);

if(compr_method == 0) {arrays[varname] = load_the_npy_file(fp);}
else {arrays[varname] = load_the_npz_array(fp,compr_bytes,uncompr_bytes);}
if(compr_method == 0) {arrays.push_back({ varname,load_the_npy_file(fp)});}
else { arrays.push_back({ varname, load_the_npz_array(fp,compr_bytes,uncompr_bytes)});}
}

fclose(fp);
Expand All @@ -284,18 +303,21 @@ cnpy::NpyArray cnpy::npz_load(std::string fname, std::string varname) {
while(1) {
std::vector<char> local_header(30);
size_t header_res = fread(&local_header[0],sizeof(char),30,fp);
if(header_res != 30)
if(header_res != 30){
fclose(fp);
throw std::runtime_error("npz_load: failed fread");

}
//if we've reached the global header, stop reading
if(local_header[2] != 0x03 || local_header[3] != 0x04) break;

//read in the variable name
uint16_t name_len = *(uint16_t*) &local_header[26];
std::string vname(name_len,' ');
size_t vname_res = fread(&vname[0],sizeof(char),name_len,fp);
if(vname_res != name_len)
size_t vname_res = fread(&vname[0],sizeof(char),name_len,fp);
if (vname_res != name_len) {
fclose(fp);
throw std::runtime_error("npz_load: failed fread");
}
vname.erase(vname.end()-4,vname.end()); //erase the lagging .npy

//read in the extra field
Expand Down
17 changes: 10 additions & 7 deletions thirdparty/cnpy/cnpy.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ namespace cnpy {
size_t num_vals;
};

using npz_t = std::map<std::string, NpyArray>;
using npz_t = std::vector<std::pair<std::string, NpyArray>>;

char BigEndianTest();
char map_type(const std::type_info& t);
Expand Down Expand Up @@ -154,6 +154,7 @@ namespace cnpy {
global_header.resize(global_header_size);
size_t res = fread(&global_header[0],sizeof(char),global_header_size,fp);
if(res != global_header_size){
fclose(fp);
throw std::runtime_error("npz_save: header read error while adding to existing zip");
}
fseek(fp,global_header_offset,SEEK_SET);
Expand Down Expand Up @@ -212,12 +213,14 @@ namespace cnpy {
footer += (uint16_t) 0; //zip file comment length

//write everything
fwrite(&local_header[0],sizeof(char),local_header.size(),fp);
fwrite(&npy_header[0],sizeof(char),npy_header.size(),fp);
fwrite(data,sizeof(T),nels,fp);
fwrite(&global_header[0],sizeof(char),global_header.size(),fp);
fwrite(&footer[0],sizeof(char),footer.size(),fp);
fclose(fp);
if (fp) {
fwrite(&local_header[0], sizeof(char), local_header.size(), fp);
fwrite(&npy_header[0], sizeof(char), npy_header.size(), fp);
fwrite(data, sizeof(T), nels, fp);
fwrite(&global_header[0], sizeof(char), global_header.size(), fp);
fwrite(&footer[0], sizeof(char), footer.size(), fp);
fclose(fp);
}
}

template<typename T> void npy_save(std::string fname, const std::vector<T> data, std::string mode = "w") {
Expand Down
18 changes: 0 additions & 18 deletions thirdparty/zlib/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,10 @@
PROJECT(zlib)


if(NOT WIN32)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-all")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-all")
endif()





if (MSVC)
# Build with multiple processes
add_definitions(/MP)
Expand All @@ -20,18 +15,10 @@ if (MSVC)

endif (MSVC)





set(BUILD_SHARED_LIBS OFF)

set(TARGET_NAME "zlib")

include_directories("${CMAKE_CURRENT_SOURCE_DIR}/zlib")

#file(GLOB lib_srcs *.c)

set(lib_srcs
zlib/adler32.c
zlib/compress.c
Expand Down Expand Up @@ -62,12 +49,8 @@ set(lib_hdrs
zlib/zutil.h
)


#file(GLOB lib_hdrs *.h)
set(lib_ext_hdrs "zlib/zlib.h" "zlib/zconf.h")

add_library(${TARGET_NAME} STATIC ${lib_srcs} ${lib_hdrs} ${lib_ext_hdrs})

target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/zlib"
"${CMAKE_CURRENT_SOURCE_DIR}/zlib/..")

Expand All @@ -81,5 +64,4 @@ if(UNIX)
endif()
endif()


set_target_properties(zlib PROPERTIES FOLDER thirdparty)
1 change: 1 addition & 0 deletions thirdparty/zlib/zlib
Submodule zlib added at cacf7f

0 comments on commit 1f6255e

Please sign in to comment.