From e4bfced13878c197563f15a94de1dfc781ad0f94 Mon Sep 17 00:00:00 2001 From: Harshil Goel Date: Fri, 31 Jan 2025 04:13:07 +0530 Subject: [PATCH 1/4] uids interesect --- posting/list.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/posting/list.go b/posting/list.go index 34f75880c93..13d80e18be5 100644 --- a/posting/list.go +++ b/posting/list.go @@ -1712,8 +1712,21 @@ func (l *List) Uids(opt ListOptions) (*pb.List, error) { // Pre-assign length to make it faster. l.RLock() // Use approximate length for initial capacity. - res := make([]uint64, 0, l.mutationMap.len()+codec.ApproxLen(l.plist.Pack)) + res := make([]uint64, 0, l.ApproxLen()) out := &pb.List{} + + if opt.Intersect != nil && len(opt.Intersect) < l.ApproxLen() { + for _, uid := range opt.Intersect { + ok, _, err := l.findPosting(uid, opt.ReadTs) + if err != nil { + return nil, err + } + res = append(res, p.Uid) + } + out.Uids = res + return out, nil + } + if l.mutationMap.len() == 0 && opt.Intersect != nil && len(l.plist.Splits) == 0 { if opt.ReadTs < l.minTs { l.RUnlock() @@ -1724,9 +1737,22 @@ func (l *List) Uids(opt ListOptions) (*pb.List, error) { return out, nil } + var uidMin, uidMax uint64 = 0, 0 + if opt.Intersect != nil && len(opt.Intersect.Uids) > 0 { + uidMin = opt.Intersect.Uids[0] + uidMax = opt.Intersect.Uids[len(opt.Intersect.Uids)-1] + } + err := l.iterate(opt.ReadTs, opt.AfterUid, func(p *pb.Posting) error { if p.PostingType == pb.Posting_REF { + if p.Uid < uidMin { + return nil + } + if p.Uid > uidMax && uidMax > 0 { + return ErrStopIteration + } res = append(res, p.Uid) + if opt.First < 0 { // We need the last N. // TODO: This could be optimized by only considering some of the last UidBlocks. From 6cf35b5f8bdb010b8792a0d010e06418b38512e8 Mon Sep 17 00:00:00 2001 From: Harshil Goel Date: Fri, 31 Jan 2025 04:17:48 +0530 Subject: [PATCH 2/4] fixed build --- posting/list.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/posting/list.go b/posting/list.go index 13d80e18be5..7e06db881d7 100644 --- a/posting/list.go +++ b/posting/list.go @@ -1715,13 +1715,15 @@ func (l *List) Uids(opt ListOptions) (*pb.List, error) { res := make([]uint64, 0, l.ApproxLen()) out := &pb.List{} - if opt.Intersect != nil && len(opt.Intersect) < l.ApproxLen() { - for _, uid := range opt.Intersect { + if opt.Intersect != nil && len(opt.Intersect.Uids) < l.ApproxLen() { + for _, uid := range opt.Intersect.Uids { ok, _, err := l.findPosting(uid, opt.ReadTs) if err != nil { return nil, err } - res = append(res, p.Uid) + if ok { + res = append(res, uid) + } } out.Uids = res return out, nil From d5ee815afb07c2ebccd8840b2cf1f0e4d683a165 Mon Sep 17 00:00:00 2001 From: Harshil Goel Date: Fri, 31 Jan 2025 05:12:55 +0530 Subject: [PATCH 3/4] fixed locks --- posting/list.go | 111 +++++++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/posting/list.go b/posting/list.go index 7e06db881d7..653a4b0947b 100644 --- a/posting/list.go +++ b/posting/list.go @@ -1709,72 +1709,79 @@ func (l *List) Uids(opt ListOptions) (*pb.List, error) { if opt.First == 0 { opt.First = math.MaxInt32 } - // Pre-assign length to make it faster. - l.RLock() - // Use approximate length for initial capacity. - res := make([]uint64, 0, l.ApproxLen()) - out := &pb.List{} - if opt.Intersect != nil && len(opt.Intersect.Uids) < l.ApproxLen() { - for _, uid := range opt.Intersect.Uids { - ok, _, err := l.findPosting(uid, opt.ReadTs) - if err != nil { - return nil, err - } - if ok { - res = append(res, uid) + getUidList := func() (*pb.List, error, bool) { + // Pre-assign length to make it faster. + l.RLock() + defer l.RUnlock() + // Use approximate length for initial capacity. + res := make([]uint64, 0, l.ApproxLen()) + out := &pb.List{} + + if opt.Intersect != nil && len(opt.Intersect.Uids) < l.ApproxLen() { + for _, uid := range opt.Intersect.Uids { + ok, _, err := l.findPosting(uid, opt.ReadTs) + if err != nil { + return nil, err, false + } + if ok { + res = append(res, uid) + } } + out.Uids = res + return out, nil, false } - out.Uids = res - return out, nil - } - if l.mutationMap.len() == 0 && opt.Intersect != nil && len(l.plist.Splits) == 0 { - if opt.ReadTs < l.minTs { - l.RUnlock() - return out, errors.Wrapf(ErrTsTooOld, "While reading UIDs") + if l.mutationMap.len() == 0 && opt.Intersect != nil && len(l.plist.Splits) == 0 { + if opt.ReadTs < l.minTs { + return out, errors.Wrapf(ErrTsTooOld, "While reading UIDs") + } + algo.IntersectCompressedWith(l.plist.Pack, opt.AfterUid, opt.Intersect, out) + return out, nil, false } - algo.IntersectCompressedWith(l.plist.Pack, opt.AfterUid, opt.Intersect, out) - l.RUnlock() - return out, nil - } - var uidMin, uidMax uint64 = 0, 0 - if opt.Intersect != nil && len(opt.Intersect.Uids) > 0 { - uidMin = opt.Intersect.Uids[0] - uidMax = opt.Intersect.Uids[len(opt.Intersect.Uids)-1] - } + var uidMin, uidMax uint64 = 0, 0 + if opt.Intersect != nil && len(opt.Intersect.Uids) > 0 { + uidMin = opt.Intersect.Uids[0] + uidMax = opt.Intersect.Uids[len(opt.Intersect.Uids)-1] + } - err := l.iterate(opt.ReadTs, opt.AfterUid, func(p *pb.Posting) error { - if p.PostingType == pb.Posting_REF { - if p.Uid < uidMin { - return nil - } - if p.Uid > uidMax && uidMax > 0 { - return ErrStopIteration - } - res = append(res, p.Uid) + err := l.iterate(opt.ReadTs, opt.AfterUid, func(p *pb.Posting) error { + if p.PostingType == pb.Posting_REF { + if p.Uid < uidMin { + return nil + } + if p.Uid > uidMax && uidMax > 0 { + return ErrStopIteration + } + res = append(res, p.Uid) - if opt.First < 0 { - // We need the last N. - // TODO: This could be optimized by only considering some of the last UidBlocks. - if len(res) > -opt.First { - res = res[1:] + if opt.First < 0 { + // We need the last N. + // TODO: This could be optimized by only considering some of the last UidBlocks. + if len(res) > -opt.First { + res = res[1:] + } + } else if len(res) > opt.First { + return ErrStopIteration } - } else if len(res) > opt.First { - return ErrStopIteration } + return nil + }) + if err != nil { + return out, errors.Wrapf(err, "cannot retrieve UIDs from list with key %s", + hex.EncodeToString(l.key)), false } - return nil - }) - l.RUnlock() - if err != nil { - return out, errors.Wrapf(err, "cannot retrieve UIDs from list with key %s", - hex.EncodeToString(l.key)) + out.Uids = res + return out, nil, true } // Do The intersection here as it's optimized. - out.Uids = res + out, err, applyIntersectWith := getUidList() + if err != nil || !applyIntersectWith { + return out, err + } + lenBefore := len(res) if opt.Intersect != nil { algo.IntersectWith(out, opt.Intersect, out) From 8d0c9fd2217c9b71baded63247efdfc21dff6556 Mon Sep 17 00:00:00 2001 From: Harshil Goel Date: Fri, 31 Jan 2025 05:58:43 +0530 Subject: [PATCH 4/4] first --- posting/list.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/posting/list.go b/posting/list.go index 653a4b0947b..a6e40336bb9 100644 --- a/posting/list.go +++ b/posting/list.go @@ -1734,7 +1734,7 @@ func (l *List) Uids(opt ListOptions) (*pb.List, error) { if l.mutationMap.len() == 0 && opt.Intersect != nil && len(l.plist.Splits) == 0 { if opt.ReadTs < l.minTs { - return out, errors.Wrapf(ErrTsTooOld, "While reading UIDs") + return out, errors.Wrapf(ErrTsTooOld, "While reading UIDs"), false } algo.IntersectCompressedWith(l.plist.Pack, opt.AfterUid, opt.Intersect, out) return out, nil, false @@ -1782,7 +1782,7 @@ func (l *List) Uids(opt ListOptions) (*pb.List, error) { return out, err } - lenBefore := len(res) + lenBefore := len(out.Uids) if opt.Intersect != nil { algo.IntersectWith(out, opt.Intersect, out) }