Skip to content

Commit

Permalink
[section_faiss_vector_index] Minor refactor removing unnecessary vars (
Browse files Browse the repository at this point in the history
  • Loading branch information
abhinavdangeti authored Feb 13, 2025
1 parent 24c0c83 commit 8187adc
Showing 1 changed file with 9 additions and 26 deletions.
35 changes: 9 additions & 26 deletions section_faiss_vector_index.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,17 +146,13 @@ func (v *faissVectorIndexSection) Merge(opaque map[int]resetable, segments []*Se
// remap the docID from the old segment to the new document nos.
// provided. furthermore, also drop the now-invalid doc nums
// of that segment
var vecIDNotDeleted bool // indicates if the vector ID was not deleted.
var newDocID uint64 // new docID in the new segment
if newDocNumsIn[segI][uint32(docID)] != docDropped {
newDocID = newDocNumsIn[segI][uint32(docID)]
vecIDNotDeleted = true
}
// if the remapped doc ID is valid, track it
// as part of vecs to be reconstructed (for larger indexes).
// this would account only the valid vector IDs, so the deleted
// ones won't be reconstructed in the final index.
if vecIDNotDeleted {
newDocID := newDocNumsIn[segI][uint32(docID)]

// if the remapped doc ID is valid, track it
// as part of vecs to be reconstructed (for larger indexes).
// this would account only the valid vector IDs, so the deleted
// ones won't be reconstructed in the final index.
vecToDocID[vecID] = newDocID
indexes[curIdx].vecIds = append(indexes[curIdx].vecIds, vecID)
}
Expand Down Expand Up @@ -250,11 +246,7 @@ func (v *vectorIndexOpaque) flushVectorIndex(indexBytes []byte, w *CountHashWrit

// write the vector index data
_, err = w.Write(indexBytes)
if err != nil {
return err
}

return nil
return err
}

// Divide the estimated nprobe with this value to optimize
Expand Down Expand Up @@ -379,7 +371,6 @@ func (v *vectorIndexOpaque) mergeAndWriteVectorIndexes(sbs []*SegmentBase,
// the reconstructed ones anymore and doing so will hold up memory which can
// be detrimental while creating indexes during introduction.
freeReconstructedIndexes(vecIndexes)
vecIndexes = nil

faissIndex, err := faiss.IndexFactory(dims, indexDescription, metric)
if err != nil {
Expand Down Expand Up @@ -414,20 +405,12 @@ func (v *vectorIndexOpaque) mergeAndWriteVectorIndexes(sbs []*SegmentBase,
return err
}

indexData = nil
finalVecIDs = nil
var mergedIndexBytes []byte
mergedIndexBytes, err = faiss.WriteIndexIntoBuffer(faissIndex)
if err != nil {
return err
}

err = v.flushVectorIndex(mergedIndexBytes, w)
mergedIndexBytes, err := faiss.WriteIndexIntoBuffer(faissIndex)
if err != nil {
return err
}

return nil
return v.flushVectorIndex(mergedIndexBytes, w)
}

// todo: can be parallelized.
Expand Down

0 comments on commit 8187adc

Please sign in to comment.