diff --git a/.gitmodules b/.gitmodules index f00c50cb834b5f..dad6d39352d7b0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -21,3 +21,4 @@ [submodule "thirdparty/zlib/zlib"] path = thirdparty/zlib/zlib url = https://github.com/madler/zlib.git + ignore = dirty diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index cdd6ab87df5ef5..2c6b656408b6f6 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -6,4 +6,5 @@ add_subdirectory(ittapi) add_subdirectory(itt_collector) add_subdirectory(xbyak EXCLUDE_FROM_ALL) add_subdirectory(zlib EXCLUDE_FROM_ALL) +add_subdirectory(cnpy EXCLUDE_FROM_ALL) openvino_developer_export_targets(COMPONENT openvino_common TARGETS xbyak) diff --git a/thirdparty/cnpy/CMakeLists.txt b/thirdparty/cnpy/CMakeLists.txt index 9eb550f053143f..1e90f758d073f5 100644 --- a/thirdparty/cnpy/CMakeLists.txt +++ b/thirdparty/cnpy/CMakeLists.txt @@ -5,26 +5,19 @@ endif(COMMAND cmake_policy) project(CNPY) +set(BUILD_SHARED_LIBS OFF) +set(TARGET_NAME "cnpy") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") +add_library(cnpy STATIC "cnpy.cpp") -option(ENABLE_STATIC "Build static (.a) library" ON) +if(NOT WIN32) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-all") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-all") + target_compile_options(${TARGET_NAME} PUBLIC -Wno-unused-variable) +endif() -find_package(ZLIB REQUIRED) +target_link_libraries(${TARGET_NAME} PUBLIC zlib) +target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/..") -include_directories(${ZLIB_INCLUDE_DIRS}) - -add_library(cnpy SHARED "cnpy.cpp") -target_link_libraries(cnpy ${ZLIB_LIBRARIES}) -install(TARGETS "cnpy" LIBRARY DESTINATION lib PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) - -if(ENABLE_STATIC) - add_library(cnpy-static STATIC "cnpy.cpp") - set_target_properties(cnpy-static PROPERTIES OUTPUT_NAME "cnpy") - install(TARGETS "cnpy-static" ARCHIVE DESTINATION lib) -endif(ENABLE_STATIC) - -install(FILES "cnpy.h" DESTINATION include) -install(FILES "mat2npz" "npy2mat" "npz2mat" DESTINATION bin PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) - -add_executable(example1 example1.cpp) -target_link_libraries(example1 cnpy) +set_target_properties(cnpy PROPERTIES FOLDER thirdparty) \ No newline at end of file diff --git a/thirdparty/cnpy/cnpy.cpp b/thirdparty/cnpy/cnpy.cpp index 2d28578643b109..a3a3e0ef406660 100644 --- a/thirdparty/cnpy/cnpy.cpp +++ b/thirdparty/cnpy/cnpy.cpp @@ -103,10 +103,17 @@ void cnpy::parse_npy_header(unsigned char* buffer,size_t& word_size, std::vector void cnpy::parse_npy_header(FILE* fp, size_t& word_size, std::vector& shape, bool& fortran_order) { char buffer[256]; - size_t res = fread(buffer,sizeof(char),11,fp); - if(res != 11) + std::string header; + size_t res = fread(buffer,sizeof(char),11,fp); + if (res != 11) throw std::runtime_error("parse_npy_header: failed fread"); - std::string header = fgets(buffer,256,fp); + char * data = fgets(buffer, 256, fp); + if (data != NULL) { + header = data; + } + else { + header = ""; + } assert(header[header.size()-1] == '\n'); size_t loc1, loc2; @@ -180,12 +187,16 @@ cnpy::NpyArray load_the_npy_file(FILE* fp) { size_t word_size; bool fortran_order; cnpy::parse_npy_header(fp,word_size,shape,fortran_order); - - cnpy::NpyArray arr(shape, word_size, fortran_order); - size_t nread = fread(arr.data(),1,arr.num_bytes(),fp); - if(nread != arr.num_bytes()) - throw std::runtime_error("load_the_npy_file: failed fread"); - return arr; + if (word_size >= 0 && word_size < ULLONG_MAX) { + cnpy::NpyArray arr(shape, word_size, fortran_order); + size_t nread = fread(arr.data(), 1, arr.num_bytes(), fp); + if (nread != arr.num_bytes()) + throw std::runtime_error("load_the_npy_file: failed fread"); + return arr; + } + else { + throw std::runtime_error("load_the_npy_file: incorrect word_size"); + } } cnpy::NpyArray load_the_npz_array(FILE* fp, uint32_t compr_bytes, uint32_t uncompr_bytes) { @@ -218,13 +229,17 @@ cnpy::NpyArray load_the_npz_array(FILE* fp, uint32_t compr_bytes, uint32_t uncom size_t word_size; bool fortran_order; cnpy::parse_npy_header(&buffer_uncompr[0],word_size,shape,fortran_order); + if (word_size >= 0 && word_size < ULLONG_MAX) { + cnpy::NpyArray array(shape, word_size, fortran_order); - cnpy::NpyArray array(shape, word_size, fortran_order); - - size_t offset = uncompr_bytes - array.num_bytes(); - memcpy(array.data(),&buffer_uncompr[0]+offset,array.num_bytes()); + size_t offset = uncompr_bytes - array.num_bytes(); + memcpy(array.data(), &buffer_uncompr[0] + offset, array.num_bytes()); - return array; + return array; + } + else { + throw std::runtime_error("load_the_npz_array: incorrect word_size"); + } } cnpy::npz_t cnpy::npz_load(std::string fname) { @@ -239,9 +254,10 @@ cnpy::npz_t cnpy::npz_load(std::string fname) { while(1) { std::vector local_header(30); size_t headerres = fread(&local_header[0],sizeof(char),30,fp); - if(headerres != 30) + if (headerres != 30) { + fclose(fp); throw std::runtime_error("npz_load: failed fread"); - + } //if we've reached the global header, stop reading if(local_header[2] != 0x03 || local_header[3] != 0x04) break; @@ -249,9 +265,10 @@ cnpy::npz_t cnpy::npz_load(std::string fname) { uint16_t name_len = *(uint16_t*) &local_header[26]; std::string varname(name_len,' '); size_t vname_res = fread(&varname[0],sizeof(char),name_len,fp); - if(vname_res != name_len) + if (vname_res != name_len) { + fclose(fp); throw std::runtime_error("npz_load: failed fread"); - + } //erase the lagging .npy varname.erase(varname.end()-4,varname.end()); @@ -260,16 +277,18 @@ cnpy::npz_t cnpy::npz_load(std::string fname) { if(extra_field_len > 0) { std::vector buff(extra_field_len); size_t efield_res = fread(&buff[0],sizeof(char),extra_field_len,fp); - if(efield_res != extra_field_len) + if (efield_res != extra_field_len) { + fclose(fp); throw std::runtime_error("npz_load: failed fread"); + } } uint16_t compr_method = *reinterpret_cast(&local_header[0]+8); uint32_t compr_bytes = *reinterpret_cast(&local_header[0]+18); uint32_t uncompr_bytes = *reinterpret_cast(&local_header[0]+22); - if(compr_method == 0) {arrays[varname] = load_the_npy_file(fp);} - else {arrays[varname] = load_the_npz_array(fp,compr_bytes,uncompr_bytes);} + if(compr_method == 0) {arrays.push_back({ varname,load_the_npy_file(fp)});} + else { arrays.push_back({ varname, load_the_npz_array(fp,compr_bytes,uncompr_bytes)});} } fclose(fp); @@ -284,18 +303,21 @@ cnpy::NpyArray cnpy::npz_load(std::string fname, std::string varname) { while(1) { std::vector local_header(30); size_t header_res = fread(&local_header[0],sizeof(char),30,fp); - if(header_res != 30) + if(header_res != 30){ + fclose(fp); throw std::runtime_error("npz_load: failed fread"); - + } //if we've reached the global header, stop reading if(local_header[2] != 0x03 || local_header[3] != 0x04) break; //read in the variable name uint16_t name_len = *(uint16_t*) &local_header[26]; std::string vname(name_len,' '); - size_t vname_res = fread(&vname[0],sizeof(char),name_len,fp); - if(vname_res != name_len) + size_t vname_res = fread(&vname[0],sizeof(char),name_len,fp); + if (vname_res != name_len) { + fclose(fp); throw std::runtime_error("npz_load: failed fread"); + } vname.erase(vname.end()-4,vname.end()); //erase the lagging .npy //read in the extra field diff --git a/thirdparty/cnpy/cnpy.h b/thirdparty/cnpy/cnpy.h index 0d3bb4c3c2e992..750251f480eaf4 100644 --- a/thirdparty/cnpy/cnpy.h +++ b/thirdparty/cnpy/cnpy.h @@ -60,7 +60,7 @@ namespace cnpy { size_t num_vals; }; - using npz_t = std::map; + using npz_t = std::vector>; char BigEndianTest(); char map_type(const std::type_info& t); @@ -154,6 +154,7 @@ namespace cnpy { global_header.resize(global_header_size); size_t res = fread(&global_header[0],sizeof(char),global_header_size,fp); if(res != global_header_size){ + fclose(fp); throw std::runtime_error("npz_save: header read error while adding to existing zip"); } fseek(fp,global_header_offset,SEEK_SET); @@ -212,12 +213,14 @@ namespace cnpy { footer += (uint16_t) 0; //zip file comment length //write everything - fwrite(&local_header[0],sizeof(char),local_header.size(),fp); - fwrite(&npy_header[0],sizeof(char),npy_header.size(),fp); - fwrite(data,sizeof(T),nels,fp); - fwrite(&global_header[0],sizeof(char),global_header.size(),fp); - fwrite(&footer[0],sizeof(char),footer.size(),fp); - fclose(fp); + if (fp) { + fwrite(&local_header[0], sizeof(char), local_header.size(), fp); + fwrite(&npy_header[0], sizeof(char), npy_header.size(), fp); + fwrite(data, sizeof(T), nels, fp); + fwrite(&global_header[0], sizeof(char), global_header.size(), fp); + fwrite(&footer[0], sizeof(char), footer.size(), fp); + fclose(fp); + } } template void npy_save(std::string fname, const std::vector data, std::string mode = "w") { diff --git a/thirdparty/zlib/CMakeLists.txt b/thirdparty/zlib/CMakeLists.txt index 13d6b5d2d5fe78..e77a00ff50bfab 100644 --- a/thirdparty/zlib/CMakeLists.txt +++ b/thirdparty/zlib/CMakeLists.txt @@ -1,15 +1,10 @@ PROJECT(zlib) - if(NOT WIN32) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-all") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-all") endif() - - - - if (MSVC) # Build with multiple processes add_definitions(/MP) @@ -20,18 +15,10 @@ if (MSVC) endif (MSVC) - - - - set(BUILD_SHARED_LIBS OFF) - set(TARGET_NAME "zlib") - include_directories("${CMAKE_CURRENT_SOURCE_DIR}/zlib") -#file(GLOB lib_srcs *.c) - set(lib_srcs zlib/adler32.c zlib/compress.c @@ -62,12 +49,8 @@ set(lib_hdrs zlib/zutil.h ) - -#file(GLOB lib_hdrs *.h) set(lib_ext_hdrs "zlib/zlib.h" "zlib/zconf.h") - add_library(${TARGET_NAME} STATIC ${lib_srcs} ${lib_hdrs} ${lib_ext_hdrs}) - target_include_directories(${TARGET_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/zlib" "${CMAKE_CURRENT_SOURCE_DIR}/zlib/..") @@ -81,5 +64,4 @@ if(UNIX) endif() endif() - set_target_properties(zlib PROPERTIES FOLDER thirdparty) \ No newline at end of file diff --git a/thirdparty/zlib/zlib b/thirdparty/zlib/zlib new file mode 160000 index 00000000000000..cacf7f1d4e3d44 --- /dev/null +++ b/thirdparty/zlib/zlib @@ -0,0 +1 @@ +Subproject commit cacf7f1d4e3d44d871b605da3b647f07d718623f