Skip to content

Commit

Permalink
Formatting fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@daxpryce
daxpryce committed Jul 12, 2023
1 parent 7561009 commit 9042595
Show file tree
Hide file tree
Showing 15 changed files with 155 additions and 164 deletions.
2 changes: 1 addition & 1 deletion apps/build_memory_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ int main(int argc, char **argv)
try
{
std::cout << "Starting index build with R: " << R << " Lbuild: " << L << " alpha: " << alpha
<< " #threads: " << num_threads << std::endl;
<< " #threads: " << num_threads << std::endl;

size_t data_num, data_dim;
diskann::get_bin_metadata(data_path, data_num, data_dim);
Expand Down
14 changes: 7 additions & 7 deletions apps/range_search_disk_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,17 +165,17 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre

std::string recall_string = "Recall@rng=" + std::to_string(search_range);
std::cout << std::setw(6) << "L" << std::setw(12) << "Beamwidth" << std::setw(16) << "QPS" << std::setw(16)
<< "Mean Latency" << std::setw(16) << "99.9 Latency" << std::setw(16) << "Mean IOs" << std::setw(16)
<< "CPU (s)";
<< "Mean Latency" << std::setw(16) << "99.9 Latency" << std::setw(16) << "Mean IOs" << std::setw(16)
<< "CPU (s)";
if (calc_recall_flag)
{
std::cout << std::setw(16) << recall_string << std::endl;
}
else
std::cout << std::endl;
std::cout << "==============================================================="
"==========================================="
<< std::endl;
"==========================================="
<< std::endl;

std::vector<std::vector<std::vector<uint32_t>>> query_result_ids(Lvec.size());

Expand Down Expand Up @@ -247,9 +247,9 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
ratio_of_sums = (1.0 * total_true_positive) / (1.0 * total_positive);
}

std::cout << std::setw(6) << L << std::setw(12) << optimized_beamwidth << std::setw(16) << qps
<< std::setw(16) << mean_latency << std::setw(16) << latency_999 << std::setw(16) << mean_ios
<< std::setw(16) << mean_cpuus;
std::cout << std::setw(6) << L << std::setw(12) << optimized_beamwidth << std::setw(16) << qps << std::setw(16)
<< mean_latency << std::setw(16) << latency_999 << std::setw(16) << mean_ios << std::setw(16)
<< mean_cpuus;
if (calc_recall_flag)
{
std::cout << std::setw(16) << recall << "," << ratio_of_sums << std::endl;
Expand Down
14 changes: 7 additions & 7 deletions apps/search_disk_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,17 +177,17 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre

std::string recall_string = "Recall@" + std::to_string(recall_at);
std::cout << std::setw(6) << "L" << std::setw(12) << "Beamwidth" << std::setw(16) << "QPS" << std::setw(16)
<< "Mean Latency" << std::setw(16) << "99.9 Latency" << std::setw(16) << "Mean IOs" << std::setw(16)
<< "CPU (s)";
<< "Mean Latency" << std::setw(16) << "99.9 Latency" << std::setw(16) << "Mean IOs" << std::setw(16)
<< "CPU (s)";
if (calc_recall_flag)
{
std::cout << std::setw(16) << recall_string << std::endl;
}
else
std::cout << std::endl;
std::cout << "==============================================================="
"======================================================="
<< std::endl;
"======================================================="
<< std::endl;

std::vector<std::vector<uint32_t>> query_result_ids(Lvec.size());
std::vector<std::vector<float>> query_result_dists(Lvec.size());
Expand Down Expand Up @@ -277,9 +277,9 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
best_recall = std::max(recall, best_recall);
}

std::cout << std::setw(6) << L << std::setw(12) << optimized_beamwidth << std::setw(16) << qps
<< std::setw(16) << mean_latency << std::setw(16) << latency_999 << std::setw(16) << mean_ios
<< std::setw(16) << mean_cpuus;
std::cout << std::setw(6) << L << std::setw(12) << optimized_beamwidth << std::setw(16) << qps << std::setw(16)
<< mean_latency << std::setw(16) << latency_999 << std::setw(16) << mean_ios << std::setw(16)
<< mean_cpuus;
if (calc_recall_flag)
{
std::cout << std::setw(16) << recall << std::endl;
Expand Down
8 changes: 4 additions & 4 deletions apps/test_streaming_scenario.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,13 +133,13 @@ void delete_and_consolidate(diskann::AbstractIndex &index, diskann::IndexWritePa
if (report._status == diskann::consolidation_report::status_code::LOCK_FAIL)
{
std::cerr << "Unable to acquire consolidate delete lock after "
<< "deleting points " << start << " to " << end << ". Will retry in " << wait_time
<< "seconds." << std::endl;
<< "deleting points " << start << " to " << end << ". Will retry in " << wait_time
<< "seconds." << std::endl;
}
else if (report._status == diskann::consolidation_report::status_code::INCONSISTENT_COUNT_ERROR)
{
std::cerr << "Inconsistent counts in data structure. "
<< "Will retry in " << wait_time << "seconds." << std::endl;
<< "Will retry in " << wait_time << "seconds." << std::endl;
}
else
{
Expand Down Expand Up @@ -197,7 +197,7 @@ void build_incremental_index(const std::string &data_path, const uint32_t L, con

diskann::get_bin_metadata(data_path, num_points, dim);
std::cout << "metadata: file " << data_path << " has " << num_points << " points in " << dim << " dims"
<< std::endl;
<< std::endl;
aligned_dim = ROUND_UP(dim, 8);

auto index_config = diskann::IndexConfigBuilder()
Expand Down
2 changes: 1 addition & 1 deletion include/cached_io.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ class cached_ifstream
cache_buf = new char[cacheSize];
reader.read(cache_buf, cacheSize);
std::cout << "Opened: " << filename.c_str() << ", size: " << fsize << ", cache_size: " << cacheSize
<< std::endl;
<< std::endl;
}
catch (std::system_error &e)
{
Expand Down
28 changes: 13 additions & 15 deletions include/utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,10 +139,10 @@ inline int delete_file(const std::string &fileName)
if (rc != 0)
{
std::cerr << "Could not delete file: " << fileName
<< " even though it exists. This might indicate a permissions "
"issue. "
"If you see this message, please contact the diskann team."
<< std::endl;
<< " even though it exists. This might indicate a permissions "
"issue. "
"If you see this message, please contact the diskann team."
<< std::endl;
}
return rc;
}
Expand Down Expand Up @@ -249,8 +249,8 @@ inline void realloc_aligned(void **ptr, size_t size, size_t align)
*ptr = ::_aligned_realloc(*ptr, size, align);
#else
std::cerr << "No aligned realloc on GCC. Must malloc and mem_align, "
"left it out for now."
<< std::endl;
"left it out for now."
<< std::endl;
#endif
if (*ptr == nullptr)
report_memory_allocation_failure();
Expand Down Expand Up @@ -701,8 +701,7 @@ inline size_t save_bin(const std::string &filename, T *data, size_t npts, size_t
size_t bytes_written = npts * ndims * sizeof(T) + 2 * sizeof(uint32_t);
writer.write((char *)&npts_i32, sizeof(int));
writer.write((char *)&ndims_i32, sizeof(int));
std::cout << "bin: #pts = " << npts << ", #dims = " << ndims << ", size = " << bytes_written << "B"
<< std::endl;
std::cout << "bin: #pts = " << npts << ", #dims = " << ndims << ", size = " << bytes_written << "B" << std::endl;

writer.write((char *)data, npts * ndims * sizeof(T));
writer.close();
Expand Down Expand Up @@ -743,7 +742,7 @@ inline void load_aligned_bin_impl(std::basic_istream<char> &reader, size_t actua
}
rounded_dim = ROUND_UP(dim, 8);
std::cout << "Metadata: #pts = " << npts << ", #dims = " << dim << ", aligned_dim = " << rounded_dim << "... "
<< std::flush;
<< std::flush;
size_t allocSize = npts * rounded_dim * sizeof(T);
std::cout << "allocating aligned memory of " << allocSize << " bytes... " << std::flush;
alloc_aligned(((void **)&data), allocSize, 8 * sizeof(T));
Expand Down Expand Up @@ -939,7 +938,7 @@ inline void copy_aligned_data_from_file(const char *bin_file, T *&data, size_t &
if (data == nullptr)
{
std::cerr << "Memory was not allocated for " << data << " before calling the load function. Exiting..."
<< std::endl;
<< std::endl;
throw diskann::ANNException("Null pointer passed to copy_aligned_data_from_file function", -1, __FUNCSIG__,
__FILE__, __LINE__);
}
Expand Down Expand Up @@ -1020,7 +1019,7 @@ inline bool validate_index_file_size(std::ifstream &in)
if (actual_file_size != expected_file_size)
{
std::cerr << "Index file size error. Expected size (metadata): " << expected_file_size
<< ", actual file size : " << actual_file_size << "." << std::endl;
<< ", actual file size : " << actual_file_size << "." << std::endl;
return false;
}
return true;
Expand Down Expand Up @@ -1189,10 +1188,9 @@ inline void printProcessMemory(const char *message)
PROCESS_MEMORY_COUNTERS counters;
HANDLE h = GetCurrentProcess();
GetProcessMemoryInfo(h, &counters, sizeof(counters));
std::cout << message
<< " [Peaking Working Set size: " << counters.PeakWorkingSetSize * 1.0 / (1024.0 * 1024 * 1024)
<< "GB Working set size: " << counters.WorkingSetSize * 1.0 / (1024.0 * 1024 * 1024)
<< "GB Private bytes " << counters.PagefileUsage * 1.0 / (1024 * 1024 * 1024) << "GB]" << std::endl;
std::cout << message << " [Peaking Working Set size: " << counters.PeakWorkingSetSize * 1.0 / (1024.0 * 1024 * 1024)
<< "GB Working set size: " << counters.WorkingSetSize * 1.0 / (1024.0 * 1024 * 1024)
<< "GB Private bytes " << counters.PagefileUsage * 1.0 / (1024 * 1024 * 1024) << "GB]" << std::endl;
}
#else

Expand Down
48 changes: 24 additions & 24 deletions src/disk_utils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,8 @@ size_t calculate_num_pq_chunks(double final_index_ram_limit, size_t points_num,
else
{
std::cout << "Compress ratio: " << compress_ratio << " #new pq_chunks: " << chunks_by_cr
<< " is either zero or greater than num_pq_chunks: " << num_pq_chunks
<< ". num_pq_chunks is unchanged. " << std::endl;
<< " is either zero or greater than num_pq_chunks: " << num_pq_chunks
<< ". num_pq_chunks is unchanged. " << std::endl;
}
}
else
Expand All @@ -127,16 +127,16 @@ size_t calculate_num_pq_chunks(double final_index_ram_limit, size_t points_num,
num_pq_chunks = num_pq_chunks > MAX_PQ_CHUNKS ? MAX_PQ_CHUNKS : num_pq_chunks;

std::cout << "Compressing " << dim << "-dimensional data into " << num_pq_chunks << " bytes per vector."
<< std::endl;
<< std::endl;
return num_pq_chunks;
}

template <typename T> T *generateRandomWarmup(uint64_t warmup_num, uint64_t warmup_dim, uint64_t warmup_aligned_dim)
{
T *warmup = nullptr;
warmup_num = 100000;
std::cout << "Generating random warmup file with dim " << warmup_dim << " and aligned dim "
<< warmup_aligned_dim << std::flush;
std::cout << "Generating random warmup file with dim " << warmup_dim << " and aligned dim " << warmup_aligned_dim
<< std::flush;
diskann::alloc_aligned(((void **)&warmup), warmup_num * warmup_aligned_dim * sizeof(T), 8 * sizeof(T));
std::memset(warmup, 0, warmup_num * warmup_aligned_dim * sizeof(T));
std::random_device rd;
Expand Down Expand Up @@ -165,8 +165,8 @@ T *load_warmup(MemoryMappedFiles &files, const std::string &cache_warmup_file, u
{
diskann::load_aligned_bin<T>(files, cache_warmup_file, warmup, warmup_num, file_dim, file_aligned_dim);
std::cout << "In the warmup file: " << cache_warmup_file << " File dim: " << file_dim
<< " File aligned dim: " << file_aligned_dim << " Expected dim: " << warmup_dim
<< " Expected aligned dim: " << warmup_aligned_dim << std::endl;
<< " File aligned dim: " << file_aligned_dim << " Expected dim: " << warmup_dim
<< " Expected aligned dim: " << warmup_aligned_dim << std::endl;

if (file_dim != warmup_dim || file_aligned_dim != warmup_aligned_dim)
{
Expand Down Expand Up @@ -535,7 +535,7 @@ void breakup_dense_points(const std::string data_file, const std::string labels_
}
}
std::cout << "fraction of dense points with >= " << density << " labels = " << (float)dense_pts / (float)npts
<< std::endl;
<< std::endl;

if (labels_per_point.size() != 0)
{
Expand Down Expand Up @@ -625,8 +625,8 @@ int build_merged_vamana_index(std::string base_file, diskann::Metric compareMetr
// TODO: Make this honest when there is filter support
if (full_index_ram < ram_budget * 1024 * 1024 * 1024)
{
std::cout << "Full index fits in RAM budget, should consume at most "
<< full_index_ram / (1024 * 1024 * 1024) << "GiBs, so building in one shot" << std::endl;
std::cout << "Full index fits in RAM budget, should consume at most " << full_index_ram / (1024 * 1024 * 1024)
<< "GiBs, so building in one shot" << std::endl;

diskann::IndexWriteParameters paras = diskann::IndexWriteParametersBuilder(L, R)
.with_filter_list_size(Lf)
Expand Down Expand Up @@ -1037,18 +1037,18 @@ int build_disk_index(const char *dataFilePath, const char *indexFilePath, const
if (param_list.size() < 5 || param_list.size() > 9)
{
std::cout << "Correct usage of parameters is R (max degree)\n"
"L (indexing list size, better if >= R)\n"
"B (RAM limit of final index in GB)\n"
"M (memory limit while indexing)\n"
"T (number of threads for indexing)\n"
"B' (PQ bytes for disk index: optional parameter for "
"very large dimensional data)\n"
"reorder (set true to include full precision in data file"
": optional paramter, use only when using disk PQ\n"
"build_PQ_byte (number of PQ bytes for inde build; set 0 to use "
"full precision vectors)\n"
"QD Quantized Dimension to overwrite the derived dim from B "
<< std::endl;
"L (indexing list size, better if >= R)\n"
"B (RAM limit of final index in GB)\n"
"M (memory limit while indexing)\n"
"T (number of threads for indexing)\n"
"B' (PQ bytes for disk index: optional parameter for "
"very large dimensional data)\n"
"reorder (set true to include full precision in data file"
": optional paramter, use only when using disk PQ\n"
"build_PQ_byte (number of PQ bytes for inde build; set 0 to use "
"full precision vectors)\n"
"QD Quantized Dimension to overwrite the derived dim from B "
<< std::endl;
return -1;
}

Expand Down Expand Up @@ -1164,7 +1164,7 @@ int build_disk_index(const char *dataFilePath, const char *indexFilePath, const
}

std::cout << "Starting index build: R=" << R << " L=" << L << " Query RAM budget: " << final_index_ram_limit
<< " Indexing ram budget: " << indexing_ram_budget << " T: " << num_threads << std::endl;
<< " Indexing ram budget: " << indexing_ram_budget << " T: " << num_threads << std::endl;

auto s = std::chrono::high_resolution_clock::now();

Expand Down Expand Up @@ -1216,7 +1216,7 @@ int build_disk_index(const char *dataFilePath, const char *indexFilePath, const
}

std::cout << "Compressing " << dim << "-dimensional data into " << num_pq_chunks << " bytes per vector."
<< std::endl;
<< std::endl;

generate_quantized_data<T>(data_file_to_use, pq_pivots_path, pq_compressed_vectors_path, compareMetric, p_val,
num_pq_chunks, use_opq, codebook_prefix);
Expand Down
30 changes: 15 additions & 15 deletions src/distance.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -627,15 +627,15 @@ template <> diskann::Distance<float> *get_distance_function(diskann::Metric m)
else if (m == diskann::Metric::INNER_PRODUCT)
{
std::cout << "Inner product: Using AVX2 implementation "
"AVXDistanceInnerProductFloat"
<< std::endl;
"AVXDistanceInnerProductFloat"
<< std::endl;
return new diskann::AVXDistanceInnerProductFloat();
}
else if (m == diskann::Metric::FAST_L2)
{
std::cout << "Fast_L2: Using AVX2 implementation with norm "
"memoization DistanceFastL2<float>"
<< std::endl;
"memoization DistanceFastL2<float>"
<< std::endl;
return new diskann::DistanceFastL2<float>();
}
else
Expand Down Expand Up @@ -666,16 +666,16 @@ template <> diskann::Distance<int8_t> *get_distance_function(diskann::Metric m)
else
{
std::cout << "Older CPU. Using slow distance computation "
"SlowDistanceL2Int<int8_t>."
<< std::endl;
"SlowDistanceL2Int<int8_t>."
<< std::endl;
return new diskann::SlowDistanceL2<int8_t>();
}
}
else if (m == diskann::Metric::COSINE)
{
std::cout << "Using either AVX or AVX2 for Cosine similarity "
"DistanceCosineInt8."
<< std::endl;
"DistanceCosineInt8."
<< std::endl;
return new diskann::DistanceCosineInt8();
}
else
Expand All @@ -693,19 +693,19 @@ template <> diskann::Distance<uint8_t> *get_distance_function(diskann::Metric m)
{
#ifdef _WINDOWS
std::cout << "WARNING: AVX/AVX2 distance function not defined for Uint8. "
"Using "
"slow version. "
"Contact [email protected] if you need AVX/AVX2 support."
<< std::endl;
"Using "
"slow version. "
"Contact [email protected] if you need AVX/AVX2 support."
<< std::endl;
#endif
return new diskann::DistanceL2UInt8();
}
else if (m == diskann::Metric::COSINE)
{
std::cout << "AVX/AVX2 distance function not defined for Uint8. Using "
"slow version SlowDistanceCosineUint8() "
"Contact [email protected] if you need AVX/AVX2 support."
<< std::endl;
"slow version SlowDistanceCosineUint8() "
"Contact [email protected] if you need AVX/AVX2 support."
<< std::endl;
return new diskann::SlowDistanceCosineUInt8();
}
else
Expand Down
Loading

0 comments on commit 9042595

Please sign in to comment.