From fbb2d44d97a40261b37873587d371a079ad776f3 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 08:58:35 -0700 Subject: [PATCH 01/21] in progress --- .../Shared/Extensions/SourceTextExtensions.cs | 39 ++++++++++++++++--- 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 47d9178365398..fdd0d7b04bfb7 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -4,7 +4,9 @@ using System; using System.Collections.Immutable; +using System.Diagnostics; using System.IO; +using System.Linq; using System.Text; using System.Threading; using Microsoft.CodeAnalysis.Host; @@ -17,9 +19,12 @@ namespace Microsoft.CodeAnalysis.Shared.Extensions; internal static partial class SourceTextExtensions { - // char pooled memory : 8K * 256 = 2MB + // char pooled memory: 4k characters. 4K * 256 * 2 (bytes per char) = 2MB private const int CharArrayLength = 4 * 1024; + // 16k characters. Equivalent to 32KB in memory. comes from SourceText char buffer size and less than large object size + public const int SourceTextLengthThreshold = 32 * 1024 / sizeof(char); + /// /// Note: there is a strong invariant that you only get arrays back from this that are exactly long. Putting arrays back into this of the wrong length will result in broken @@ -168,9 +173,6 @@ public static int IndexOfNonWhiteSpace(this SourceText text, int start, int leng return -1; } - // 32KB. comes from SourceText char buffer size and less than large object size - internal const int SourceTextLengthThreshold = 32 * 1024 / sizeof(char); - public static void WriteTo(this SourceText sourceText, ObjectWriter writer, CancellationToken cancellationToken) { // Source length @@ -234,6 +236,31 @@ public static SourceText ReadFrom(ITextFactoryService textService, ObjectReader return textService.CreateText(textReader, encoding, checksumAlgorithm, cancellationToken); } + private static ImmutableArray CreateChunks(int totalLength) + { + var numberOfChunks = 1 + (totalLength / CharArrayLength); + var buffer = new FixedSizeArrayBuilder(numberOfChunks); + for (var i = 0; i < numberOfChunks; i++) + buffer.Add(s_charArrayPool.Allocate()); + + return buffer.MoveToImmutable(); + } + + public static SourceText CreateSourceText( + SyntaxNode node, Encoding? encoding, SourceHashAlgorithm checksumAlgorithm, CancellationToken cancellationToken) + { + var totalLength = node.FullWidth(); + + var chunks = CreateChunks(totalLength); + + } + + public static SourceText ReadFromChunks(ImmutableArray chunks, int totalLength) + { + Debug.Assert(chunks.All(static c => c.Length == CharArrayLength)); + return SourceText.From(new ObjectReaderTextReader(chunks, CharArrayLength, totalLength)); + } + private sealed class ObjectReaderTextReader : TextReaderWithLength { private readonly ImmutableArray _chunks; @@ -256,7 +283,7 @@ public static TextReader Create(ObjectReader reader) var numberOfChunks = reader.ReadInt32(); // read as chunks - using var _ = ArrayBuilder.GetInstance(numberOfChunks, out var chunks); + var chunks = new FixedSizeArrayBuilder(numberOfChunks); var offset = 0; for (var i = 0; i < numberOfChunks; i++) @@ -279,7 +306,7 @@ public static TextReader Create(ObjectReader reader) } Contract.ThrowIfFalse(offset == length); - return new ObjectReaderTextReader(chunks.ToImmutableAndClear(), chunkSize, length); + return new ObjectReaderTextReader(chunks.MoveToImmutable(), chunkSize, length); } private ObjectReaderTextReader(ImmutableArray chunks, int chunkSize, int length) From 87b3c8bbc48b1234a2d6c18d40622ba3d6ced9d7 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 09:57:09 -0700 Subject: [PATCH 02/21] Add helper --- .../Shared/Extensions/SourceTextExtensions.cs | 80 +++++++++++++++---- 1 file changed, 66 insertions(+), 14 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index fdd0d7b04bfb7..30248b09a78cc 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -19,18 +19,21 @@ namespace Microsoft.CodeAnalysis.Shared.Extensions; internal static partial class SourceTextExtensions { - // char pooled memory: 4k characters. 4K * 256 * 2 (bytes per char) = 2MB - private const int CharArrayLength = 4 * 1024; + private const int SegmentShift = 12; + private const int SegmentMask = 0b_1111_1111_1111; + + // char segment: 4k characters. 4K * 256 * 2 (bytes per char) = 2MB + private const int CharSegmentLength = 1 << SegmentShift; // 16k characters. Equivalent to 32KB in memory. comes from SourceText char buffer size and less than large object size public const int SourceTextLengthThreshold = 32 * 1024 / sizeof(char); /// /// Note: there is a strong invariant that you only get arrays back from this that are exactly long. Putting arrays back into this of the wrong length will result in broken + /// cref="CharSegmentLength"/> long. Putting arrays back into this of the wrong length will result in broken /// behavior. Do not expose this pool outside of this class. /// - private static readonly ObjectPool s_charArrayPool = new(() => new char[CharArrayLength], 256); + private static readonly ObjectPool s_charArrayPool = new(() => new char[CharSegmentLength], 256); public static void GetLineAndOffset(this SourceText text, int position, out int lineNumber, out int offset) { @@ -196,10 +199,10 @@ private static void WriteChunksTo(SourceText sourceText, ObjectWriter writer, in // chunk size using var pooledObject = s_charArrayPool.GetPooledObject(); var buffer = pooledObject.Object; - Contract.ThrowIfTrue(buffer.Length != CharArrayLength); + Contract.ThrowIfTrue(buffer.Length != CharSegmentLength); // We write out the chunk size for sanity purposes. - writer.WriteInt32(CharArrayLength); + writer.WriteInt32(CharSegmentLength); // number of chunks var numberOfChunks = 1 + (length / buffer.Length); @@ -238,7 +241,7 @@ public static SourceText ReadFrom(ITextFactoryService textService, ObjectReader private static ImmutableArray CreateChunks(int totalLength) { - var numberOfChunks = 1 + (totalLength / CharArrayLength); + var numberOfChunks = 1 + (totalLength / CharSegmentLength); var buffer = new FixedSizeArrayBuilder(numberOfChunks); for (var i = 0; i < numberOfChunks; i++) buffer.Add(s_charArrayPool.Allocate()); @@ -249,16 +252,65 @@ private static ImmutableArray CreateChunks(int totalLength) public static SourceText CreateSourceText( SyntaxNode node, Encoding? encoding, SourceHashAlgorithm checksumAlgorithm, CancellationToken cancellationToken) { + // If this node is small enough to not go into the LOH, we can just fast path directly to creating a SourceText from it. var totalLength = node.FullWidth(); + if (totalLength <= SourceTextLengthThreshold) + return SourceText.From(node.ToFullString(), encoding, checksumAlgorithm); + + encoding ??= Encoding.UTF8; var chunks = CreateChunks(totalLength); + var chunkWriter = new CharArrayChunkTextWriter(totalLength, chunks, encoding); + node.WriteTo(chunkWriter); + Contract.ThrowIfTrue(totalLength != chunkWriter.Position); + + } + + private sealed class CharArrayChunkTextWriter : TextWriter + { + private readonly int _totalLength; + private readonly ImmutableArray _chunks; + public int Position; + + public CharArrayChunkTextWriter(int totalLength, ImmutableArray chunks, Encoding encoding) + { + _totalLength = totalLength; + _chunks = chunks; + Encoding = encoding; + } + + public override Encoding Encoding { get; } + + public override void Write(string? value) + { + Contract.ThrowIfNull(value); + if (value == "") + return; + + var valueSpan = value.AsSpan(); + while (valueSpan.Length > 0) + { + var chunk = _chunks[Position >> SegmentShift]; + + var chunkIndex = Position & SegmentMask; + Contract.ThrowIfTrue(chunkIndex >= CharSegmentLength); + + var count = Math.Min(valueSpan.Length, CharSegmentLength - chunkIndex); + valueSpan.Slice(0, count).CopyTo(chunk.AsSpan().Slice(chunkIndex, count)); + + Position += count; + valueSpan = valueSpan.Slice(count); + } + + Contract.ThrowIfTrue(Position > _totalLength); + } } public static SourceText ReadFromChunks(ImmutableArray chunks, int totalLength) { - Debug.Assert(chunks.All(static c => c.Length == CharArrayLength)); - return SourceText.From(new ObjectReaderTextReader(chunks, CharArrayLength, totalLength)); + Debug.Assert(chunks.All(static c => c.Length == CharSegmentLength)); + return SourceText.From(new ObjectReaderTextReader(chunks, CharSegmentLength, totalLength)); } private sealed class ObjectReaderTextReader : TextReaderWithLength @@ -279,7 +331,7 @@ public static TextReader Create(ObjectReader reader) } var chunkSize = reader.ReadInt32(); - Contract.ThrowIfTrue(chunkSize != CharArrayLength); + Contract.ThrowIfTrue(chunkSize != CharSegmentLength); var numberOfChunks = reader.ReadInt32(); // read as chunks @@ -292,14 +344,14 @@ public static TextReader Create(ObjectReader reader) var (currentChunk, currentChunkLength) = reader.ReadCharArray( static length => { - Contract.ThrowIfTrue(length > CharArrayLength); + Contract.ThrowIfTrue(length > CharSegmentLength); return s_charArrayPool.Allocate(); }); - Contract.ThrowIfTrue(currentChunk.Length != CharArrayLength); + Contract.ThrowIfTrue(currentChunk.Length != CharSegmentLength); // All but the last chunk must be completely filled. - Contract.ThrowIfTrue(i < numberOfChunks - 1 && currentChunkLength != CharArrayLength); + Contract.ThrowIfTrue(i < numberOfChunks - 1 && currentChunkLength != CharSegmentLength); chunks.Add(currentChunk); offset += currentChunkLength; @@ -315,7 +367,7 @@ private ObjectReaderTextReader(ImmutableArray chunks, int chunkSize, int _chunks = chunks; _chunkSize = chunkSize; _disposed = false; - Contract.ThrowIfTrue(chunkSize != CharArrayLength); + Contract.ThrowIfTrue(chunkSize != CharSegmentLength); Contract.ThrowIfTrue(chunks.Any(static (c, s) => c.Length != s, chunkSize)); } From 56e4d1380dc46cf5104d015a2e4059e316e7f648 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 12:23:19 -0700 Subject: [PATCH 03/21] in progress --- .../Core/Portable/Shared/Extensions/SourceTextExtensions.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 30248b09a78cc..696379a90f196 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -285,8 +285,6 @@ public CharArrayChunkTextWriter(int totalLength, ImmutableArray chunks, public override void Write(string? value) { Contract.ThrowIfNull(value); - if (value == "") - return; var valueSpan = value.AsSpan(); while (valueSpan.Length > 0) From 8384a7f42c3d9f85ad74ae5d9d8a69cc49aba344 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:00:53 -0700 Subject: [PATCH 04/21] in progress --- .../Shared/Extensions/SourceTextExtensions.cs | 100 ++++++++---------- 1 file changed, 47 insertions(+), 53 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 696379a90f196..2d6ce43f530ac 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -234,53 +234,47 @@ private static void WriteChunksTo(SourceText sourceText, ObjectWriter writer, in public static SourceText ReadFrom(ITextFactoryService textService, ObjectReader reader, Encoding? encoding, SourceHashAlgorithm checksumAlgorithm, CancellationToken cancellationToken) { - using var textReader = ObjectReaderTextReader.Create(reader); + using var textReader = CharArrayChunkTextReader.CreateFromObjectReader(reader); return textService.CreateText(textReader, encoding, checksumAlgorithm, cancellationToken); } - private static ImmutableArray CreateChunks(int totalLength) - { - var numberOfChunks = 1 + (totalLength / CharSegmentLength); - var buffer = new FixedSizeArrayBuilder(numberOfChunks); - for (var i = 0; i < numberOfChunks; i++) - buffer.Add(s_charArrayPool.Allocate()); - - return buffer.MoveToImmutable(); - } - public static SourceText CreateSourceText( - SyntaxNode node, Encoding? encoding, SourceHashAlgorithm checksumAlgorithm, CancellationToken cancellationToken) + ITextFactoryService textService, SyntaxNode node, Encoding? encoding, SourceHashAlgorithm checksumAlgorithm, CancellationToken cancellationToken) { // If this node is small enough to not go into the LOH, we can just fast path directly to creating a SourceText from it. var totalLength = node.FullWidth(); if (totalLength <= SourceTextLengthThreshold) return SourceText.From(node.ToFullString(), encoding, checksumAlgorithm); - encoding ??= Encoding.UTF8; - var chunks = CreateChunks(totalLength); - var chunkWriter = new CharArrayChunkTextWriter(totalLength, chunks, encoding); + using var chunkWriter = new CharArrayChunkTextWriter(totalLength, chunks, encoding!); node.WriteTo(chunkWriter); Contract.ThrowIfTrue(totalLength != chunkWriter.Position); - } + using var chunkReader = new CharArrayChunkTextReader(chunks, CharSegmentLength, totalLength); + var result = textService.CreateText(chunkReader, encoding, checksumAlgorithm, cancellationToken); - private sealed class CharArrayChunkTextWriter : TextWriter - { - private readonly int _totalLength; - private readonly ImmutableArray _chunks; - public int Position; - public CharArrayChunkTextWriter(int totalLength, ImmutableArray chunks, Encoding encoding) + static ImmutableArray CreateChunks(int totalLength) { - _totalLength = totalLength; - _chunks = chunks; - Encoding = encoding; + var numberOfChunks = 1 + (totalLength / CharSegmentLength); + var buffer = new FixedSizeArrayBuilder(numberOfChunks); + for (var i = 0; i < numberOfChunks; i++) + buffer.Add(s_charArrayPool.Allocate()); + + return buffer.MoveToImmutable(); } + } - public override Encoding Encoding { get; } + private sealed class CharArrayChunkTextWriter(int totalLength, ImmutableArray chunks, Encoding encoding) : TextWriter + { + private readonly int _totalLength = totalLength; + private readonly ImmutableArray _chunks = chunks; + public int Position; + + public override Encoding Encoding { get; } = encoding; public override void Write(string? value) { @@ -305,21 +299,31 @@ public override void Write(string? value) } } - public static SourceText ReadFromChunks(ImmutableArray chunks, int totalLength) - { - Debug.Assert(chunks.All(static c => c.Length == CharSegmentLength)); - return SourceText.From(new ObjectReaderTextReader(chunks, CharSegmentLength, totalLength)); - } + //public static SourceText ReadFromChunks(ImmutableArray chunks, int totalLength) + //{ + // Debug.Assert(chunks.All(static c => c.Length == CharSegmentLength)); + // return SourceText.From(new ObjectReaderTextReader(chunks, CharSegmentLength, totalLength)); + //} - private sealed class ObjectReaderTextReader : TextReaderWithLength + private sealed class CharArrayChunkTextReader : TextReaderWithLength { private readonly ImmutableArray _chunks; private readonly int _chunkSize; private bool _disposed; - private int _position; + public int Position; + + public CharArrayChunkTextReader(ImmutableArray chunks, int chunkSize, int length) + : base(length) + { + _chunks = chunks; + _chunkSize = chunkSize; + _disposed = false; + Contract.ThrowIfTrue(chunkSize != CharSegmentLength); + Contract.ThrowIfTrue(chunks.Any(static (c, s) => c.Length != s, chunkSize)); + } - public static TextReader Create(ObjectReader reader) + public static TextReader CreateFromObjectReader(ObjectReader reader) { var length = reader.ReadInt32(); if (length < SourceTextLengthThreshold) @@ -356,17 +360,7 @@ public static TextReader Create(ObjectReader reader) } Contract.ThrowIfFalse(offset == length); - return new ObjectReaderTextReader(chunks.MoveToImmutable(), chunkSize, length); - } - - private ObjectReaderTextReader(ImmutableArray chunks, int chunkSize, int length) - : base(length) - { - _chunks = chunks; - _chunkSize = chunkSize; - _disposed = false; - Contract.ThrowIfTrue(chunkSize != CharSegmentLength); - Contract.ThrowIfTrue(chunks.Any(static (c, s) => c.Length != s, chunkSize)); + return new CharArrayChunkTextReader(chunks.MoveToImmutable(), chunkSize, length); } protected override void Dispose(bool disposing) @@ -385,22 +379,22 @@ protected override void Dispose(bool disposing) public override int Peek() { - if (_position >= Length) + if (Position >= Length) { return -1; } - return Read(_position); + return Read(Position); } public override int Read() { - if (_position >= Length) + if (Position >= Length) { return -1; } - return Read(_position++); + return Read(Position++); } public override int Read(char[] buffer, int index, int count) @@ -427,11 +421,11 @@ public override int Read(char[] buffer, int index, int count) } // adjust to actual char to read - var totalCharsToRead = Math.Min(count, Length - _position); + var totalCharsToRead = Math.Min(count, Length - Position); count = totalCharsToRead; - var chunkIndex = GetIndexFromPosition(_position); - var chunkStartOffset = GetColumnFromPosition(_position); + var chunkIndex = GetIndexFromPosition(Position); + var chunkStartOffset = GetColumnFromPosition(Position); while (true) { @@ -451,7 +445,7 @@ public override int Read(char[] buffer, int index, int count) chunkIndex++; } - _position += totalCharsToRead; + Position += totalCharsToRead; return totalCharsToRead; } From 65b3b71f9939d5f3fb2c55d330ba86cef5e1318c Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:04:13 -0700 Subject: [PATCH 05/21] Cleanup --- .../Shared/Extensions/SourceTextExtensions.cs | 41 ++++++++----------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 2d6ce43f530ac..70e01a1eee82e 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -19,11 +19,8 @@ namespace Microsoft.CodeAnalysis.Shared.Extensions; internal static partial class SourceTextExtensions { - private const int SegmentShift = 12; - private const int SegmentMask = 0b_1111_1111_1111; - - // char segment: 4k characters. 4K * 256 * 2 (bytes per char) = 2MB - private const int CharSegmentLength = 1 << SegmentShift; + // char segment: 4k characters. 4K * 256 * 2 (bytes per char) = 4MB + private const int CharSegmentLength = 4096; // 16k characters. Equivalent to 32KB in memory. comes from SourceText char buffer size and less than large object size public const int SourceTextLengthThreshold = 32 * 1024 / sizeof(char); @@ -253,9 +250,11 @@ public static SourceText CreateSourceText( node.WriteTo(chunkWriter); Contract.ThrowIfTrue(totalLength != chunkWriter.Position); - using var chunkReader = new CharArrayChunkTextReader(chunks, CharSegmentLength, totalLength); + using var chunkReader = new CharArrayChunkTextReader(chunks, totalLength); var result = textService.CreateText(chunkReader, encoding, checksumAlgorithm, cancellationToken); + Contract.ThrowIfTrue(totalLength != chunkReader.Position); + return result; static ImmutableArray CreateChunks(int totalLength) { @@ -268,6 +267,9 @@ static ImmutableArray CreateChunks(int totalLength) } } + private static int GetIndexFromPosition(int position) => position / CharSegmentLength; + private static int GetColumnFromPosition(int position) => position % CharSegmentLength; + private sealed class CharArrayChunkTextWriter(int totalLength, ImmutableArray chunks, Encoding encoding) : TextWriter { private readonly int _totalLength = totalLength; @@ -283,44 +285,36 @@ public override void Write(string? value) var valueSpan = value.AsSpan(); while (valueSpan.Length > 0) { - var chunk = _chunks[Position >> SegmentShift]; + var chunk = _chunks[GetIndexFromPosition(Position)]; + Contract.ThrowIfTrue(chunk.Length != CharSegmentLength); - var chunkIndex = Position & SegmentMask; + var chunkIndex = GetColumnFromPosition(Position); Contract.ThrowIfTrue(chunkIndex >= CharSegmentLength); var count = Math.Min(valueSpan.Length, CharSegmentLength - chunkIndex); - valueSpan.Slice(0, count).CopyTo(chunk.AsSpan().Slice(chunkIndex, count)); + valueSpan[..count].CopyTo(chunk.AsSpan().Slice(chunkIndex, count)); Position += count; - valueSpan = valueSpan.Slice(count); + valueSpan = valueSpan[count..]; } Contract.ThrowIfTrue(Position > _totalLength); } } - //public static SourceText ReadFromChunks(ImmutableArray chunks, int totalLength) - //{ - // Debug.Assert(chunks.All(static c => c.Length == CharSegmentLength)); - // return SourceText.From(new ObjectReaderTextReader(chunks, CharSegmentLength, totalLength)); - //} - private sealed class CharArrayChunkTextReader : TextReaderWithLength { private readonly ImmutableArray _chunks; - private readonly int _chunkSize; private bool _disposed; public int Position; - public CharArrayChunkTextReader(ImmutableArray chunks, int chunkSize, int length) + public CharArrayChunkTextReader(ImmutableArray chunks, int length) : base(length) { _chunks = chunks; - _chunkSize = chunkSize; _disposed = false; - Contract.ThrowIfTrue(chunkSize != CharSegmentLength); - Contract.ThrowIfTrue(chunks.Any(static (c, s) => c.Length != s, chunkSize)); + Contract.ThrowIfTrue(chunks.Any(static (c, s) => c.Length != s, CharSegmentLength)); } public static TextReader CreateFromObjectReader(ObjectReader reader) @@ -360,7 +354,7 @@ public static TextReader CreateFromObjectReader(ObjectReader reader) } Contract.ThrowIfFalse(offset == length); - return new CharArrayChunkTextReader(chunks.MoveToImmutable(), chunkSize, length); + return new CharArrayChunkTextReader(chunks.MoveToImmutable(), length); } protected override void Dispose(bool disposing) @@ -456,8 +450,5 @@ private int Read(int position) return _chunks[chunkIndex][chunkColumn]; } - - private int GetIndexFromPosition(int position) => position / _chunkSize; - private int GetColumnFromPosition(int position) => position % _chunkSize; } } From dcdd431b7b1eb0e0b8c10da73cb0483ce39c7aa6 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:13:27 -0700 Subject: [PATCH 06/21] Add factories --- ...ntaxTreeFactoryService.ParsedSyntaxTree.cs | 33 ++++++++++++++++--- .../CSharpSyntaxTreeFactoryService.cs | 26 +++++++-------- .../AbstractSyntaxTreeFactoryService.cs | 3 -- ...ntaxTreeFactoryService.ParsedSyntaxTree.vb | 28 +++++++++++++--- .../VisualBasicSyntaxTreeFactoryService.vb | 25 +++++++++++--- 5 files changed, 84 insertions(+), 31 deletions(-) diff --git a/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs b/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs index 2d002177327b1..93cbc12009ec6 100644 --- a/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs +++ b/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs @@ -5,6 +5,8 @@ using System.Diagnostics.CodeAnalysis; using System.Text; using System.Threading; +using Microsoft.CodeAnalysis.Host; +using Microsoft.CodeAnalysis.Shared.Extensions; using Microsoft.CodeAnalysis.Text; namespace Microsoft.CodeAnalysis.CSharp; @@ -18,6 +20,7 @@ private sealed class ParsedSyntaxTree : CSharpSyntaxTree { private readonly CSharpSyntaxNode _root; private readonly SourceHashAlgorithm _checksumAlgorithm; + private readonly ITextFactoryService _textFactoryService; public override Encoding? Encoding { get; } public override CSharpParseOptions Options { get; } @@ -25,12 +28,19 @@ private sealed class ParsedSyntaxTree : CSharpSyntaxTree private SourceText? _lazyText; - public ParsedSyntaxTree(SourceText? lazyText, CSharpSyntaxNode root, CSharpParseOptions options, string filePath, Encoding? encoding, SourceHashAlgorithm checksumAlgorithm) + public ParsedSyntaxTree( + SourceText? lazyText, + CSharpSyntaxNode root, + CSharpParseOptions options, + string filePath, + Encoding? encoding, + SourceHashAlgorithm checksumAlgorithm, + ITextFactoryService textFactoryService) { _lazyText = lazyText; _root = CloneNodeAsRoot(root); _checksumAlgorithm = checksumAlgorithm; - + _textFactoryService = textFactoryService; Encoding = encoding; Options = options; FilePath = filePath; @@ -40,7 +50,9 @@ public override SourceText GetText(CancellationToken cancellationToken) { if (_lazyText == null) { - Interlocked.CompareExchange(ref _lazyText, GetRoot(cancellationToken).GetText(Encoding, _checksumAlgorithm), null); + var sourceText = SourceTextExtensions.CreateSourceText( + _textFactoryService, GetRoot(cancellationToken), Encoding, _checksumAlgorithm, cancellationToken); + Interlocked.CompareExchange(ref _lazyText, sourceText, null); } return _lazyText; @@ -68,10 +80,21 @@ public override bool TryGetRoot([NotNullWhen(true)] out CSharpSyntaxNode? root) } public override SyntaxTree WithRootAndOptions(SyntaxNode root, ParseOptions options) - => (root == _root && options == Options) ? this : new ParsedSyntaxTree((root == _root) ? _lazyText : null, (CSharpSyntaxNode)root, (CSharpParseOptions)options, FilePath, Encoding, _checksumAlgorithm); + => root == _root && options == Options + ? this + : new ParsedSyntaxTree( + root == _root ? _lazyText : null, + (CSharpSyntaxNode)root, + (CSharpParseOptions)options, + FilePath, + Encoding, + _checksumAlgorithm, + _textFactoryService); public override SyntaxTree WithFilePath(string path) - => (path == FilePath) ? this : new ParsedSyntaxTree(_lazyText, _root, Options, path, Encoding, _checksumAlgorithm); + => path == FilePath + ? this + : new ParsedSyntaxTree(_lazyText, _root, Options, path, Encoding, _checksumAlgorithm, _textFactoryService); public override SyntaxReference GetReference(SyntaxNode node) => new NodeSyntaxReference(node); diff --git a/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.cs b/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.cs index f07ba78829056..674653fc312d4 100644 --- a/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.cs +++ b/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.cs @@ -7,30 +7,29 @@ using System; using System.Collections.Generic; using System.Composition; -using System.Diagnostics; -using System.IO; using System.Text; using System.Threading; -using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.Host; using Microsoft.CodeAnalysis.Host.Mef; -using Microsoft.CodeAnalysis.Options; using Microsoft.CodeAnalysis.Text; -using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.CSharp; -[ExportLanguageService(typeof(ISyntaxTreeFactoryService), LanguageNames.CSharp), Shared] -internal partial class CSharpSyntaxTreeFactoryService : AbstractSyntaxTreeFactoryService +internal sealed partial class CSharpSyntaxTreeFactoryService(ITextFactoryService textFactoryService) : AbstractSyntaxTreeFactoryService { - private static readonly CSharpParseOptions _parseOptionWithLatestLanguageVersion = CSharpParseOptions.Default.WithLanguageVersion(LanguageVersion.Preview); - - [ImportingConstructor] - [Obsolete(MefConstruction.ImportingConstructorMessage, error: true)] - public CSharpSyntaxTreeFactoryService() + [ExportLanguageServiceFactory(typeof(ISyntaxTreeFactoryService), LanguageNames.CSharp), Shared] + [method: ImportingConstructor] + [method: Obsolete(MefConstruction.ImportingConstructorMessage, error: true)] + private sealed class Factory() : ILanguageServiceFactory { + public ILanguageService CreateLanguageService(HostLanguageServices languageServices) + => new CSharpSyntaxTreeFactoryService(languageServices.WorkspaceServices.GetRequiredService()); } + private static readonly CSharpParseOptions _parseOptionWithLatestLanguageVersion = CSharpParseOptions.Default.WithLanguageVersion(LanguageVersion.Preview); + + private readonly ITextFactoryService _textFactoryService = textFactoryService; + public override ParseOptions GetDefaultParseOptions() => CSharpParseOptions.Default; @@ -63,7 +62,8 @@ public override bool OptionsDifferOnlyByPreprocessorDirectives(ParseOptions opti public override SyntaxTree CreateSyntaxTree(string filePath, ParseOptions options, Encoding encoding, SourceHashAlgorithm checksumAlgorithm, SyntaxNode root) { options ??= GetDefaultParseOptions(); - return new ParsedSyntaxTree(lazyText: null, (CSharpSyntaxNode)root, (CSharpParseOptions)options, filePath, encoding, checksumAlgorithm); + return new ParsedSyntaxTree( + lazyText: null, (CSharpSyntaxNode)root, (CSharpParseOptions)options, filePath, encoding, checksumAlgorithm, _textFactoryService); } public override SyntaxTree ParseSyntaxTree(string filePath, ParseOptions options, SourceText text, CancellationToken cancellationToken) diff --git a/src/Workspaces/Core/Portable/Workspace/Host/SyntaxTreeFactory/AbstractSyntaxTreeFactoryService.cs b/src/Workspaces/Core/Portable/Workspace/Host/SyntaxTreeFactory/AbstractSyntaxTreeFactoryService.cs index 35f03538856e8..b5a0c1f372766 100644 --- a/src/Workspaces/Core/Portable/Workspace/Host/SyntaxTreeFactory/AbstractSyntaxTreeFactoryService.cs +++ b/src/Workspaces/Core/Portable/Workspace/Host/SyntaxTreeFactory/AbstractSyntaxTreeFactoryService.cs @@ -4,13 +4,10 @@ #nullable disable -using System; using System.Collections.Generic; -using System.IO; using System.Text; using System.Threading; using Microsoft.CodeAnalysis.Text; -using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.Host; diff --git a/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb b/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb index 4148c89ac8972..7c0b81d673708 100644 --- a/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb +++ b/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb @@ -5,6 +5,7 @@ Imports System.Runtime.InteropServices Imports System.Text Imports System.Threading +Imports Microsoft.CodeAnalysis.Host Imports Microsoft.CodeAnalysis.Text Namespace Microsoft.CodeAnalysis.VisualBasic @@ -17,6 +18,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Private ReadOnly _root As VisualBasicSyntaxNode Private ReadOnly _checksumAlgorithm As SourceHashAlgorithm + Private ReadOnly _textFactoryService As ITextFactoryService Public Overrides ReadOnly Property Encoding As Encoding Public Overrides ReadOnly Property Options As VisualBasicParseOptions @@ -24,11 +26,18 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Private _lazyText As SourceText - Public Sub New(lazyText As SourceText, root As VisualBasicSyntaxNode, options As VisualBasicParseOptions, filePath As String, encoding As Encoding, checksumAlgorithm As SourceHashAlgorithm) + Public Sub New( + lazyText As SourceText, + root As VisualBasicSyntaxNode, + options As VisualBasicParseOptions, + filePath As String, + encoding As Encoding, + checksumAlgorithm As SourceHashAlgorithm, + textFactoryService As ITextFactoryService) _lazyText = lazyText _root = CloneNodeAsRoot(root) _checksumAlgorithm = checksumAlgorithm - + _textFactoryService = textFactoryService Me.Encoding = encoding Me.Options = options Me.FilePath = filePath @@ -36,7 +45,9 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Public Overrides Function GetText(Optional cancellationToken As CancellationToken = Nothing) As SourceText If _lazyText Is Nothing Then - Interlocked.CompareExchange(_lazyText, GetRoot(cancellationToken).GetText(Encoding, _checksumAlgorithm), Nothing) + Dim text = SourceTextExtensions.CreateSourceText( + _textFactoryService, GetRoot(cancellationToken), Encoding, _checksumAlgorithm, cancellationToken) + Interlocked.CompareExchange(_lazyText, text, Nothing) End If Return _lazyText @@ -71,13 +82,20 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Public Overrides Function WithRootAndOptions(root As SyntaxNode, options As ParseOptions) As SyntaxTree Return If(ReferenceEquals(root, _root) AndAlso options = Me.Options, Me, - New ParsedSyntaxTree(If(ReferenceEquals(root, _root), _lazyText, Nothing), DirectCast(root, VisualBasicSyntaxNode), DirectCast(options, VisualBasicParseOptions), FilePath, Encoding, _checksumAlgorithm)) + New ParsedSyntaxTree( + If(ReferenceEquals(root, _root), _lazyText, Nothing), + DirectCast(root, VisualBasicSyntaxNode), + DirectCast(options, VisualBasicParseOptions), + FilePath, + Encoding, + _checksumAlgorithm, + _textFactoryService)) End Function Public Overrides Function WithFilePath(path As String) As SyntaxTree Return If(path = FilePath, Me, - New ParsedSyntaxTree(_lazyText, _root, Options, path, Encoding, _checksumAlgorithm)) + New ParsedSyntaxTree(_lazyText, _root, Options, path, Encoding, _checksumAlgorithm, _textFactoryService)) End Function Public Overrides Function GetReference(node As SyntaxNode) As SyntaxReference diff --git a/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.vb b/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.vb index 97b8341d7aae7..1b6bd34e9ca90 100644 --- a/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.vb +++ b/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.vb @@ -11,15 +11,23 @@ Imports Microsoft.CodeAnalysis.Host.Mef Imports Microsoft.CodeAnalysis.Text Namespace Microsoft.CodeAnalysis.VisualBasic - Partial Friend Class VisualBasicSyntaxTreeFactoryService Inherits AbstractSyntaxTreeFactoryService + + Private NotInheritable Class Factory + Implements ILanguageServiceFactory + + Public Function CreateLanguageService(languageServices As HostLanguageServices) As ILanguageService Implements ILanguageServiceFactory.CreateLanguageService + Return New VisualBasicSyntaxTreeFactoryService(languageServices.WorkspaceServices.GetRequiredService(Of ITextFactoryService)()) + End Function + End Class + Private Shared ReadOnly _parseOptionsWithLatestLanguageVersion As VisualBasicParseOptions = VisualBasicParseOptions.Default.WithLanguageVersion(LanguageVersion.Latest) + Private ReadOnly _textFactoryService As ITextFactoryService - - - Public Sub New() + Private Sub New(textFactoryService As ITextFactoryService) + _textFactoryService = textFactoryService End Sub Public Overloads Overrides Function GetDefaultParseOptions() As ParseOptions @@ -75,7 +83,14 @@ Namespace Microsoft.CodeAnalysis.VisualBasic options = GetDefaultParseOptions() End If - Return New ParsedSyntaxTree(lazyText:=Nothing, DirectCast(root, VisualBasicSyntaxNode), DirectCast(options, VisualBasicParseOptions), filePath, encoding, checksumAlgorithm) + Return New ParsedSyntaxTree( + lazyText:=Nothing, + DirectCast(root, VisualBasicSyntaxNode), + DirectCast(options, VisualBasicParseOptions), + filePath, + encoding, + checksumAlgorithm, + _textFactoryService) End Function End Class End Namespace From 502b6513bc8586a0f397cd48199ec033e0794582 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:19:02 -0700 Subject: [PATCH 07/21] rename --- .../Shared/Extensions/SourceTextExtensions.cs | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 70e01a1eee82e..e010468e9e91c 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -19,18 +19,18 @@ namespace Microsoft.CodeAnalysis.Shared.Extensions; internal static partial class SourceTextExtensions { - // char segment: 4k characters. 4K * 256 * 2 (bytes per char) = 4MB - private const int CharSegmentLength = 4096; + // char array length: 4k characters. 4K * 256 * 2 (bytes per char) = 4MB + private const int CharArrayLength = 4096; // 16k characters. Equivalent to 32KB in memory. comes from SourceText char buffer size and less than large object size public const int SourceTextLengthThreshold = 32 * 1024 / sizeof(char); /// /// Note: there is a strong invariant that you only get arrays back from this that are exactly long. Putting arrays back into this of the wrong length will result in broken + /// cref="CharArrayLength"/> long. Putting arrays back into this of the wrong length will result in broken /// behavior. Do not expose this pool outside of this class. /// - private static readonly ObjectPool s_charArrayPool = new(() => new char[CharSegmentLength], 256); + private static readonly ObjectPool s_charArrayPool = new(() => new char[CharArrayLength], 256); public static void GetLineAndOffset(this SourceText text, int position, out int lineNumber, out int offset) { @@ -196,10 +196,10 @@ private static void WriteChunksTo(SourceText sourceText, ObjectWriter writer, in // chunk size using var pooledObject = s_charArrayPool.GetPooledObject(); var buffer = pooledObject.Object; - Contract.ThrowIfTrue(buffer.Length != CharSegmentLength); + Contract.ThrowIfTrue(buffer.Length != CharArrayLength); // We write out the chunk size for sanity purposes. - writer.WriteInt32(CharSegmentLength); + writer.WriteInt32(CharArrayLength); // number of chunks var numberOfChunks = 1 + (length / buffer.Length); @@ -258,7 +258,7 @@ public static SourceText CreateSourceText( static ImmutableArray CreateChunks(int totalLength) { - var numberOfChunks = 1 + (totalLength / CharSegmentLength); + var numberOfChunks = 1 + (totalLength / CharArrayLength); var buffer = new FixedSizeArrayBuilder(numberOfChunks); for (var i = 0; i < numberOfChunks; i++) buffer.Add(s_charArrayPool.Allocate()); @@ -267,8 +267,8 @@ static ImmutableArray CreateChunks(int totalLength) } } - private static int GetIndexFromPosition(int position) => position / CharSegmentLength; - private static int GetColumnFromPosition(int position) => position % CharSegmentLength; + private static int GetIndexFromPosition(int position) => position / CharArrayLength; + private static int GetColumnFromPosition(int position) => position % CharArrayLength; private sealed class CharArrayChunkTextWriter(int totalLength, ImmutableArray chunks, Encoding encoding) : TextWriter { @@ -286,12 +286,12 @@ public override void Write(string? value) while (valueSpan.Length > 0) { var chunk = _chunks[GetIndexFromPosition(Position)]; - Contract.ThrowIfTrue(chunk.Length != CharSegmentLength); + Contract.ThrowIfTrue(chunk.Length != CharArrayLength); var chunkIndex = GetColumnFromPosition(Position); - Contract.ThrowIfTrue(chunkIndex >= CharSegmentLength); + Contract.ThrowIfTrue(chunkIndex >= CharArrayLength); - var count = Math.Min(valueSpan.Length, CharSegmentLength - chunkIndex); + var count = Math.Min(valueSpan.Length, CharArrayLength - chunkIndex); valueSpan[..count].CopyTo(chunk.AsSpan().Slice(chunkIndex, count)); Position += count; @@ -314,7 +314,7 @@ public CharArrayChunkTextReader(ImmutableArray chunks, int length) { _chunks = chunks; _disposed = false; - Contract.ThrowIfTrue(chunks.Any(static (c, s) => c.Length != s, CharSegmentLength)); + Contract.ThrowIfTrue(chunks.Any(static (c, s) => c.Length != s, CharArrayLength)); } public static TextReader CreateFromObjectReader(ObjectReader reader) @@ -327,7 +327,7 @@ public static TextReader CreateFromObjectReader(ObjectReader reader) } var chunkSize = reader.ReadInt32(); - Contract.ThrowIfTrue(chunkSize != CharSegmentLength); + Contract.ThrowIfTrue(chunkSize != CharArrayLength); var numberOfChunks = reader.ReadInt32(); // read as chunks @@ -340,14 +340,14 @@ public static TextReader CreateFromObjectReader(ObjectReader reader) var (currentChunk, currentChunkLength) = reader.ReadCharArray( static length => { - Contract.ThrowIfTrue(length > CharSegmentLength); + Contract.ThrowIfTrue(length > CharArrayLength); return s_charArrayPool.Allocate(); }); - Contract.ThrowIfTrue(currentChunk.Length != CharSegmentLength); + Contract.ThrowIfTrue(currentChunk.Length != CharArrayLength); // All but the last chunk must be completely filled. - Contract.ThrowIfTrue(i < numberOfChunks - 1 && currentChunkLength != CharSegmentLength); + Contract.ThrowIfTrue(i < numberOfChunks - 1 && currentChunkLength != CharArrayLength); chunks.Add(currentChunk); offset += currentChunkLength; From b46e963990be9efdf95bd5ed9493279ffbf729b2 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:20:03 -0700 Subject: [PATCH 08/21] revert --- .../Core/Portable/Shared/Extensions/SourceTextExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index e010468e9e91c..58d706f1332aa 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -20,7 +20,7 @@ namespace Microsoft.CodeAnalysis.Shared.Extensions; internal static partial class SourceTextExtensions { // char array length: 4k characters. 4K * 256 * 2 (bytes per char) = 4MB - private const int CharArrayLength = 4096; + private const int CharArrayLength = 4 * 1024; // 16k characters. Equivalent to 32KB in memory. comes from SourceText char buffer size and less than large object size public const int SourceTextLengthThreshold = 32 * 1024 / sizeof(char); From 09db6ea6d63d126f7c44ab03168255b4fe3e9851 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:23:21 -0700 Subject: [PATCH 09/21] Simplify --- .../Shared/Extensions/SourceTextExtensions.cs | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 58d706f1332aa..55e14542348c4 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -246,7 +246,7 @@ public static SourceText CreateSourceText( var chunks = CreateChunks(totalLength); - using var chunkWriter = new CharArrayChunkTextWriter(totalLength, chunks, encoding!); + using var chunkWriter = new CharArrayChunkTextWriter(totalLength, chunks, encoding!, cancellationToken); node.WriteTo(chunkWriter); Contract.ThrowIfTrue(totalLength != chunkWriter.Position); @@ -270,10 +270,16 @@ static ImmutableArray CreateChunks(int totalLength) private static int GetIndexFromPosition(int position) => position / CharArrayLength; private static int GetColumnFromPosition(int position) => position % CharArrayLength; - private sealed class CharArrayChunkTextWriter(int totalLength, ImmutableArray chunks, Encoding encoding) : TextWriter + private sealed class CharArrayChunkTextWriter( + int totalLength, ImmutableArray chunks, Encoding encoding, CancellationToken cancellationToken) : TextWriter { private readonly int _totalLength = totalLength; private readonly ImmutableArray _chunks = chunks; + private readonly CancellationToken _cancellationToken = cancellationToken; + + /// + /// Public so that caller can assert that writing out the text actually wrote out the full text of the node. + /// public int Position; public override Encoding Encoding { get; } = encoding; @@ -285,6 +291,8 @@ public override void Write(string? value) var valueSpan = value.AsSpan(); while (valueSpan.Length > 0) { + _cancellationToken.ThrowIfCancellationRequested(); + var chunk = _chunks[GetIndexFromPosition(Position)]; Contract.ThrowIfTrue(chunk.Length != CharArrayLength); @@ -307,6 +315,9 @@ private sealed class CharArrayChunkTextReader : TextReaderWithLength private readonly ImmutableArray _chunks; private bool _disposed; + /// + /// Public so that the caller can assert that the new SourceText read all the way to the end of this successfully. + /// public int Position; public CharArrayChunkTextReader(ImmutableArray chunks, int length) @@ -374,9 +385,7 @@ protected override void Dispose(bool disposing) public override int Peek() { if (Position >= Length) - { return -1; - } return Read(Position); } @@ -384,35 +393,33 @@ public override int Peek() public override int Read() { if (Position >= Length) - { return -1; - } return Read(Position++); } + private int Read(int position) + { + var chunkIndex = GetIndexFromPosition(position); + var chunkColumn = GetColumnFromPosition(position); + + return _chunks[chunkIndex][chunkColumn]; + } + public override int Read(char[] buffer, int index, int count) { if (buffer == null) - { throw new ArgumentNullException(nameof(buffer)); - } if (index < 0 || index >= buffer.Length) - { throw new ArgumentOutOfRangeException(nameof(index)); - } if (count < 0 || (index + count) > buffer.Length) - { throw new ArgumentOutOfRangeException(nameof(count)); - } // check quick bail out if (count == 0) - { return 0; - } // adjust to actual char to read var totalCharsToRead = Math.Min(count, Length - Position); @@ -440,15 +447,8 @@ public override int Read(char[] buffer, int index, int count) } Position += totalCharsToRead; + Contract.ThrowIfTrue(Position > Length); return totalCharsToRead; } - - private int Read(int position) - { - var chunkIndex = GetIndexFromPosition(position); - var chunkColumn = GetColumnFromPosition(position); - - return _chunks[chunkIndex][chunkColumn]; - } } } From c2c2c8ef5f106d63de52622554766d641bc5148c Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:26:01 -0700 Subject: [PATCH 10/21] Fixup --- .../Shared/Extensions/SourceTextExtensions.cs | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 55e14542348c4..69643b2dce055 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -244,17 +244,28 @@ public static SourceText CreateSourceText( if (totalLength <= SourceTextLengthThreshold) return SourceText.From(node.ToFullString(), encoding, checksumAlgorithm); + // Allocate the space to write the node into. Explicitly chunked so that nothing goes into the LOH. var chunks = CreateChunks(totalLength); + try + { + // Write the node into that temp space. + using var chunkWriter = new CharArrayChunkTextWriter(totalLength, chunks, encoding!, cancellationToken); + node.WriteTo(chunkWriter); + Contract.ThrowIfTrue(totalLength != chunkWriter.Position); - using var chunkWriter = new CharArrayChunkTextWriter(totalLength, chunks, encoding!, cancellationToken); - node.WriteTo(chunkWriter); - Contract.ThrowIfTrue(totalLength != chunkWriter.Position); - - using var chunkReader = new CharArrayChunkTextReader(chunks, totalLength); - var result = textService.CreateText(chunkReader, encoding, checksumAlgorithm, cancellationToken); - Contract.ThrowIfTrue(totalLength != chunkReader.Position); + // Call into the text service to make us a SourceText from the chunks. + using var chunkReader = new CharArrayChunkTextReader(chunks, totalLength); + var result = textService.CreateText(chunkReader, encoding, checksumAlgorithm, cancellationToken); + Contract.ThrowIfTrue(totalLength != chunkReader.Position); - return result; + return result; + } + finally + { + // Finally, free the chunks so they can be used by the next caller. + foreach (var chunk in chunks) + s_charArrayPool.Free(chunk); + } static ImmutableArray CreateChunks(int totalLength) { From 6c8d03a71fd432c1da0bdbf8d19ec39e30251ae2 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:28:58 -0700 Subject: [PATCH 11/21] Double dispose --- .../Shared/Extensions/SourceTextExtensions.cs | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 69643b2dce055..47370e8c41ea7 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -246,26 +246,19 @@ public static SourceText CreateSourceText( // Allocate the space to write the node into. Explicitly chunked so that nothing goes into the LOH. var chunks = CreateChunks(totalLength); - try - { - // Write the node into that temp space. - using var chunkWriter = new CharArrayChunkTextWriter(totalLength, chunks, encoding!, cancellationToken); - node.WriteTo(chunkWriter); - Contract.ThrowIfTrue(totalLength != chunkWriter.Position); - // Call into the text service to make us a SourceText from the chunks. - using var chunkReader = new CharArrayChunkTextReader(chunks, totalLength); - var result = textService.CreateText(chunkReader, encoding, checksumAlgorithm, cancellationToken); - Contract.ThrowIfTrue(totalLength != chunkReader.Position); + // Write the node into that temp space. + using var chunkWriter = new CharArrayChunkTextWriter(totalLength, chunks, encoding!, cancellationToken); + node.WriteTo(chunkWriter); + Contract.ThrowIfTrue(totalLength != chunkWriter.Position); - return result; - } - finally - { - // Finally, free the chunks so they can be used by the next caller. - foreach (var chunk in chunks) - s_charArrayPool.Free(chunk); - } + // Call into the text service to make us a SourceText from the chunks. Disposal of this reader will free all + // the intermediary chunks we allocated. + using var chunkReader = new CharArrayChunkTextReader(chunks, totalLength); + var result = textService.CreateText(chunkReader, encoding, checksumAlgorithm, cancellationToken); + Contract.ThrowIfTrue(totalLength != chunkReader.Position); + + return result; static ImmutableArray CreateChunks(int totalLength) { From e873caada8d9e25e44a2161f62d5482bcda2d39e Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:49:32 -0700 Subject: [PATCH 12/21] Switch ot LargeText --- .../Core/Portable/Shared/Extensions/SourceTextExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 47370e8c41ea7..916d026520778 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -255,7 +255,7 @@ public static SourceText CreateSourceText( // Call into the text service to make us a SourceText from the chunks. Disposal of this reader will free all // the intermediary chunks we allocated. using var chunkReader = new CharArrayChunkTextReader(chunks, totalLength); - var result = textService.CreateText(chunkReader, encoding, checksumAlgorithm, cancellationToken); + var result = SourceText.From(chunkReader, totalLength, encoding, checksumAlgorithm); Contract.ThrowIfTrue(totalLength != chunkReader.Position); return result; From 1fd8b148217ba54153f5b661cbf1402933572b31 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:51:52 -0700 Subject: [PATCH 13/21] REvert --- ...harpSyntaxTreeFactoryService.ParsedSyntaxTree.cs | 13 ++++--------- .../Shared/Extensions/SourceTextExtensions.cs | 2 +- ...asicSyntaxTreeFactoryService.ParsedSyntaxTree.vb | 13 ++++--------- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs b/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs index 93cbc12009ec6..f910dad739ce9 100644 --- a/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs +++ b/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs @@ -20,7 +20,6 @@ private sealed class ParsedSyntaxTree : CSharpSyntaxTree { private readonly CSharpSyntaxNode _root; private readonly SourceHashAlgorithm _checksumAlgorithm; - private readonly ITextFactoryService _textFactoryService; public override Encoding? Encoding { get; } public override CSharpParseOptions Options { get; } @@ -34,13 +33,11 @@ public ParsedSyntaxTree( CSharpParseOptions options, string filePath, Encoding? encoding, - SourceHashAlgorithm checksumAlgorithm, - ITextFactoryService textFactoryService) + SourceHashAlgorithm checksumAlgorithm) { _lazyText = lazyText; _root = CloneNodeAsRoot(root); _checksumAlgorithm = checksumAlgorithm; - _textFactoryService = textFactoryService; Encoding = encoding; Options = options; FilePath = filePath; @@ -50,8 +47,7 @@ public override SourceText GetText(CancellationToken cancellationToken) { if (_lazyText == null) { - var sourceText = SourceTextExtensions.CreateSourceText( - _textFactoryService, GetRoot(cancellationToken), Encoding, _checksumAlgorithm, cancellationToken); + var sourceText = SourceTextExtensions.CreateSourceText(GetRoot(cancellationToken), Encoding, _checksumAlgorithm, cancellationToken); Interlocked.CompareExchange(ref _lazyText, sourceText, null); } @@ -88,13 +84,12 @@ public override SyntaxTree WithRootAndOptions(SyntaxNode root, ParseOptions opti (CSharpParseOptions)options, FilePath, Encoding, - _checksumAlgorithm, - _textFactoryService); + _checksumAlgorithm); public override SyntaxTree WithFilePath(string path) => path == FilePath ? this - : new ParsedSyntaxTree(_lazyText, _root, Options, path, Encoding, _checksumAlgorithm, _textFactoryService); + : new ParsedSyntaxTree(_lazyText, _root, Options, path, Encoding, _checksumAlgorithm); public override SyntaxReference GetReference(SyntaxNode node) => new NodeSyntaxReference(node); diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 916d026520778..659b204bbdecf 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -237,7 +237,7 @@ public static SourceText ReadFrom(ITextFactoryService textService, ObjectReader } public static SourceText CreateSourceText( - ITextFactoryService textService, SyntaxNode node, Encoding? encoding, SourceHashAlgorithm checksumAlgorithm, CancellationToken cancellationToken) + SyntaxNode node, Encoding? encoding, SourceHashAlgorithm checksumAlgorithm, CancellationToken cancellationToken) { // If this node is small enough to not go into the LOH, we can just fast path directly to creating a SourceText from it. var totalLength = node.FullWidth(); diff --git a/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb b/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb index 7c0b81d673708..2fc2c60bd6ba7 100644 --- a/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb +++ b/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb @@ -5,7 +5,6 @@ Imports System.Runtime.InteropServices Imports System.Text Imports System.Threading -Imports Microsoft.CodeAnalysis.Host Imports Microsoft.CodeAnalysis.Text Namespace Microsoft.CodeAnalysis.VisualBasic @@ -18,7 +17,6 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Private ReadOnly _root As VisualBasicSyntaxNode Private ReadOnly _checksumAlgorithm As SourceHashAlgorithm - Private ReadOnly _textFactoryService As ITextFactoryService Public Overrides ReadOnly Property Encoding As Encoding Public Overrides ReadOnly Property Options As VisualBasicParseOptions @@ -32,12 +30,10 @@ Namespace Microsoft.CodeAnalysis.VisualBasic options As VisualBasicParseOptions, filePath As String, encoding As Encoding, - checksumAlgorithm As SourceHashAlgorithm, - textFactoryService As ITextFactoryService) + checksumAlgorithm As SourceHashAlgorithm) _lazyText = lazyText _root = CloneNodeAsRoot(root) _checksumAlgorithm = checksumAlgorithm - _textFactoryService = textFactoryService Me.Encoding = encoding Me.Options = options Me.FilePath = filePath @@ -46,7 +42,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Public Overrides Function GetText(Optional cancellationToken As CancellationToken = Nothing) As SourceText If _lazyText Is Nothing Then Dim text = SourceTextExtensions.CreateSourceText( - _textFactoryService, GetRoot(cancellationToken), Encoding, _checksumAlgorithm, cancellationToken) + GetRoot(cancellationToken), Encoding, _checksumAlgorithm, cancellationToken) Interlocked.CompareExchange(_lazyText, text, Nothing) End If @@ -88,14 +84,13 @@ Namespace Microsoft.CodeAnalysis.VisualBasic DirectCast(options, VisualBasicParseOptions), FilePath, Encoding, - _checksumAlgorithm, - _textFactoryService)) + _checksumAlgorithm)) End Function Public Overrides Function WithFilePath(path As String) As SyntaxTree Return If(path = FilePath, Me, - New ParsedSyntaxTree(_lazyText, _root, Options, path, Encoding, _checksumAlgorithm, _textFactoryService)) + New ParsedSyntaxTree(_lazyText, _root, Options, path, Encoding, _checksumAlgorithm)) End Function Public Overrides Function GetReference(node As SyntaxNode) As SyntaxReference From 40ba1594b85a69bde501296d2ddda2a034d593c5 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:52:11 -0700 Subject: [PATCH 14/21] REvert --- .../VisualBasicSyntaxTreeFactoryService.vb | 25 ++++--------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.vb b/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.vb index 1b6bd34e9ca90..97b8341d7aae7 100644 --- a/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.vb +++ b/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.vb @@ -11,23 +11,15 @@ Imports Microsoft.CodeAnalysis.Host.Mef Imports Microsoft.CodeAnalysis.Text Namespace Microsoft.CodeAnalysis.VisualBasic + Partial Friend Class VisualBasicSyntaxTreeFactoryService Inherits AbstractSyntaxTreeFactoryService - - Private NotInheritable Class Factory - Implements ILanguageServiceFactory - - Public Function CreateLanguageService(languageServices As HostLanguageServices) As ILanguageService Implements ILanguageServiceFactory.CreateLanguageService - Return New VisualBasicSyntaxTreeFactoryService(languageServices.WorkspaceServices.GetRequiredService(Of ITextFactoryService)()) - End Function - End Class - Private Shared ReadOnly _parseOptionsWithLatestLanguageVersion As VisualBasicParseOptions = VisualBasicParseOptions.Default.WithLanguageVersion(LanguageVersion.Latest) - Private ReadOnly _textFactoryService As ITextFactoryService - Private Sub New(textFactoryService As ITextFactoryService) - _textFactoryService = textFactoryService + + + Public Sub New() End Sub Public Overloads Overrides Function GetDefaultParseOptions() As ParseOptions @@ -83,14 +75,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic options = GetDefaultParseOptions() End If - Return New ParsedSyntaxTree( - lazyText:=Nothing, - DirectCast(root, VisualBasicSyntaxNode), - DirectCast(options, VisualBasicParseOptions), - filePath, - encoding, - checksumAlgorithm, - _textFactoryService) + Return New ParsedSyntaxTree(lazyText:=Nothing, DirectCast(root, VisualBasicSyntaxNode), DirectCast(options, VisualBasicParseOptions), filePath, encoding, checksumAlgorithm) End Function End Class End Namespace From 95023369113f79cded1e0d3be88edd38f853fd6f Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:52:51 -0700 Subject: [PATCH 15/21] REvert --- .../CSharpSyntaxTreeFactoryService.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.cs b/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.cs index 674653fc312d4..f07ba78829056 100644 --- a/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.cs +++ b/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.cs @@ -7,28 +7,29 @@ using System; using System.Collections.Generic; using System.Composition; +using System.Diagnostics; +using System.IO; using System.Text; using System.Threading; +using Microsoft.CodeAnalysis.CSharp.Syntax; using Microsoft.CodeAnalysis.Host; using Microsoft.CodeAnalysis.Host.Mef; +using Microsoft.CodeAnalysis.Options; using Microsoft.CodeAnalysis.Text; +using Roslyn.Utilities; namespace Microsoft.CodeAnalysis.CSharp; -internal sealed partial class CSharpSyntaxTreeFactoryService(ITextFactoryService textFactoryService) : AbstractSyntaxTreeFactoryService +[ExportLanguageService(typeof(ISyntaxTreeFactoryService), LanguageNames.CSharp), Shared] +internal partial class CSharpSyntaxTreeFactoryService : AbstractSyntaxTreeFactoryService { - [ExportLanguageServiceFactory(typeof(ISyntaxTreeFactoryService), LanguageNames.CSharp), Shared] - [method: ImportingConstructor] - [method: Obsolete(MefConstruction.ImportingConstructorMessage, error: true)] - private sealed class Factory() : ILanguageServiceFactory - { - public ILanguageService CreateLanguageService(HostLanguageServices languageServices) - => new CSharpSyntaxTreeFactoryService(languageServices.WorkspaceServices.GetRequiredService()); - } - private static readonly CSharpParseOptions _parseOptionWithLatestLanguageVersion = CSharpParseOptions.Default.WithLanguageVersion(LanguageVersion.Preview); - private readonly ITextFactoryService _textFactoryService = textFactoryService; + [ImportingConstructor] + [Obsolete(MefConstruction.ImportingConstructorMessage, error: true)] + public CSharpSyntaxTreeFactoryService() + { + } public override ParseOptions GetDefaultParseOptions() => CSharpParseOptions.Default; @@ -62,8 +63,7 @@ public override bool OptionsDifferOnlyByPreprocessorDirectives(ParseOptions opti public override SyntaxTree CreateSyntaxTree(string filePath, ParseOptions options, Encoding encoding, SourceHashAlgorithm checksumAlgorithm, SyntaxNode root) { options ??= GetDefaultParseOptions(); - return new ParsedSyntaxTree( - lazyText: null, (CSharpSyntaxNode)root, (CSharpParseOptions)options, filePath, encoding, checksumAlgorithm, _textFactoryService); + return new ParsedSyntaxTree(lazyText: null, (CSharpSyntaxNode)root, (CSharpParseOptions)options, filePath, encoding, checksumAlgorithm); } public override SyntaxTree ParseSyntaxTree(string filePath, ParseOptions options, SourceText text, CancellationToken cancellationToken) From 6ab93f21a17565969d1b98f5f394669e195cd85d Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:56:30 -0700 Subject: [PATCH 16/21] Update src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs --- .../CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs b/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs index f910dad739ce9..8a64e2ff71573 100644 --- a/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs +++ b/src/Workspaces/CSharp/Portable/Workspace/LanguageServices/CSharpSyntaxTreeFactoryService.ParsedSyntaxTree.cs @@ -38,6 +38,7 @@ public ParsedSyntaxTree( _lazyText = lazyText; _root = CloneNodeAsRoot(root); _checksumAlgorithm = checksumAlgorithm; + Encoding = encoding; Options = options; FilePath = filePath; From 75354effd921fd2f572b81006db142247e6bcae7 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:56:48 -0700 Subject: [PATCH 17/21] Update src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb --- .../VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb b/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb index 2fc2c60bd6ba7..751738b8b856e 100644 --- a/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb +++ b/src/Workspaces/VisualBasic/Portable/Workspace/LanguageServices/VisualBasicSyntaxTreeFactoryService.ParsedSyntaxTree.vb @@ -41,8 +41,7 @@ Namespace Microsoft.CodeAnalysis.VisualBasic Public Overrides Function GetText(Optional cancellationToken As CancellationToken = Nothing) As SourceText If _lazyText Is Nothing Then - Dim text = SourceTextExtensions.CreateSourceText( - GetRoot(cancellationToken), Encoding, _checksumAlgorithm, cancellationToken) + Dim text = SourceTextExtensions.CreateSourceText(GetRoot(cancellationToken), Encoding, _checksumAlgorithm, cancellationToken) Interlocked.CompareExchange(_lazyText, text, Nothing) End If From 45c2f71fc731e8893112608ab2c8d69ce485b742 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 14:58:35 -0700 Subject: [PATCH 18/21] Docs --- .../Core/Portable/Shared/Extensions/SourceTextExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 659b204bbdecf..0e2abf181652b 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -22,8 +22,8 @@ internal static partial class SourceTextExtensions // char array length: 4k characters. 4K * 256 * 2 (bytes per char) = 4MB private const int CharArrayLength = 4 * 1024; - // 16k characters. Equivalent to 32KB in memory. comes from SourceText char buffer size and less than large object size - public const int SourceTextLengthThreshold = 32 * 1024 / sizeof(char); + // 32k characters. Equivalent to 64KB in memory bytes. Will not be put into the LOH. + public const int SourceTextLengthThreshold = 32 * 1024; /// /// Note: there is a strong invariant that you only get arrays back from this that are exactly Date: Wed, 15 May 2024 15:03:40 -0700 Subject: [PATCH 19/21] Primary constructor --- .../Shared/Extensions/SourceTextExtensions.cs | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 0e2abf181652b..b6939875d9b55 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -254,6 +254,9 @@ public static SourceText CreateSourceText( // Call into the text service to make us a SourceText from the chunks. Disposal of this reader will free all // the intermediary chunks we allocated. + + Contract.ThrowIfTrue(chunks.Any(static (c, s) => c.Length != s, CharArrayLength)); + using var chunkReader = new CharArrayChunkTextReader(chunks, totalLength); var result = SourceText.From(chunkReader, totalLength, encoding, checksumAlgorithm); Contract.ThrowIfTrue(totalLength != chunkReader.Position); @@ -314,24 +317,16 @@ public override void Write(string? value) } } - private sealed class CharArrayChunkTextReader : TextReaderWithLength + private sealed class CharArrayChunkTextReader(ImmutableArray chunks, int length) : TextReaderWithLength(length) { - private readonly ImmutableArray _chunks; - private bool _disposed; + private readonly ImmutableArray _chunks = chunks; + private bool _disposed = false; /// /// Public so that the caller can assert that the new SourceText read all the way to the end of this successfully. /// public int Position; - public CharArrayChunkTextReader(ImmutableArray chunks, int length) - : base(length) - { - _chunks = chunks; - _disposed = false; - Contract.ThrowIfTrue(chunks.Any(static (c, s) => c.Length != s, CharArrayLength)); - } - public static TextReader CreateFromObjectReader(ObjectReader reader) { var length = reader.ReadInt32(); @@ -369,7 +364,11 @@ public static TextReader CreateFromObjectReader(ObjectReader reader) } Contract.ThrowIfFalse(offset == length); - return new CharArrayChunkTextReader(chunks.MoveToImmutable(), length); + + var chunksArray = chunks.MoveToImmutable(); + Contract.ThrowIfTrue(chunksArray.Any(static (c, s) => c.Length != s, CharArrayLength)); + + return new CharArrayChunkTextReader(chunksArray, length); } protected override void Dispose(bool disposing) From a2eb5139e7b37632f6a1c54e06b6e4fe1f4af11b Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 16:26:44 -0700 Subject: [PATCH 20/21] Make into auto props --- .../Core/Portable/Shared/Extensions/SourceTextExtensions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index b6939875d9b55..50e83b304ac99 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -287,7 +287,7 @@ private sealed class CharArrayChunkTextWriter( /// /// Public so that caller can assert that writing out the text actually wrote out the full text of the node. /// - public int Position; + public int Position { get; private set; } public override Encoding Encoding { get; } = encoding; @@ -325,7 +325,7 @@ private sealed class CharArrayChunkTextReader(ImmutableArray chunks, int /// /// Public so that the caller can assert that the new SourceText read all the way to the end of this successfully. /// - public int Position; + public int Position { get; private set; } public static TextReader CreateFromObjectReader(ObjectReader reader) { From 1f23973d16e70d7179135aefca7043ea22f70107 Mon Sep 17 00:00:00 2001 From: Cyrus Najmabadi Date: Wed, 15 May 2024 16:28:05 -0700 Subject: [PATCH 21/21] Increase size and doc properly --- .../Core/Portable/Shared/Extensions/SourceTextExtensions.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs index 50e83b304ac99..5b8433ec1ccc8 100644 --- a/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs +++ b/src/Workspaces/Core/Portable/Shared/Extensions/SourceTextExtensions.cs @@ -19,7 +19,9 @@ namespace Microsoft.CodeAnalysis.Shared.Extensions; internal static partial class SourceTextExtensions { - // char array length: 4k characters. 4K * 256 * 2 (bytes per char) = 4MB + private const int ObjectPoolCount = 1024; + + // char array length: 4k characters. 4K * 1024 (object pool count) * 2 (bytes per char) = 8MB private const int CharArrayLength = 4 * 1024; // 32k characters. Equivalent to 64KB in memory bytes. Will not be put into the LOH. @@ -30,7 +32,7 @@ internal static partial class SourceTextExtensions /// cref="CharArrayLength"/> long. Putting arrays back into this of the wrong length will result in broken /// behavior. Do not expose this pool outside of this class. /// - private static readonly ObjectPool s_charArrayPool = new(() => new char[CharArrayLength], 256); + private static readonly ObjectPool s_charArrayPool = new(() => new char[CharArrayLength], ObjectPoolCount); public static void GetLineAndOffset(this SourceText text, int position, out int lineNumber, out int offset) {