Parse large JSON module dictionaries in parallel

Core/Converters/JsonParallelDictionaryConverter.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Linq;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+using CKAN.Extensions;
+
+namespace CKAN
+{
+    /// <summary>
+    /// A converter that loads a dictionary in parallel,
+    /// use with large collections of complex objects for a speed boost
+    /// </summary>
+    public class JsonParallelDictionaryConverter<V> : JsonConverter
+    {
+        public override object ReadJson(JsonReader     reader,
+                                        Type           objectType,
+                                        object         existingValue,
+                                        JsonSerializer serializer)
+            => ParseWithProgress(JObject.Load(reader)
+                                        .Properties()
+                                        .ToArray(),
+                                 serializer);
+
+        private object ParseWithProgress(JProperty[]    properties,
+                                         JsonSerializer serializer)
+            => Partitioner.Create(properties, true)
+                          .AsParallel()
+                          .WithMergeOptions(ParallelMergeOptions.NotBuffered)
+                          .Select(prop => new KeyValuePair<string, V>(
+                                              prop.Name,
+                                              prop.Value.ToObject<V>()))
+                          .WithProgress(properties.Length,
+                                        serializer.Context.Context as IProgress<int>)
+                          .ToDictionary();
+
+        public override bool CanWrite => false;
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            throw new NotImplementedException();
+        }
+
+        // Only convert when we're an explicit attribute
+        public override bool CanConvert(Type object_type) => false;
+    }
+}

Core/Extensions/EnumerableExtensions.cs

@@ -2,6 +2,7 @@
 using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
+using System.Threading;
 
 namespace CKAN.Extensions
 {
@@ -34,6 +35,31 @@ public static Dictionary<K, V> ToDictionary<K, V>(this IEnumerable<KeyValuePair<
             => pairs.ToDictionary(kvp => kvp.Key,
                                   kvp => kvp.Value);
 
+        public static Dictionary<K, V> ToDictionary<K, V>(this ParallelQuery<KeyValuePair<K, V>> pairs)
+            => pairs.ToDictionary(kvp => kvp.Key,
+                                  kvp => kvp.Value);
+
+        // https://stackoverflow.com/a/55591477/2422988
+        public static ParallelQuery<T> WithProgress<T>(this ParallelQuery<T> source,
+                                                       long                  totalCount,
+                                                       IProgress<int>        progress)
+        {
+            long count       = 0;
+            int  prevPercent = -1;
+            return progress == null
+                ? source
+                : source.Select(item =>
+                {
+                    var percent = (int)(100 * Interlocked.Increment(ref count) / totalCount);
+                    if (percent > prevPercent)
+                    {
+                        progress.Report(percent);
+                        prevPercent = percent;
+                    }
+                    return item;
+                });
+        }
+
         public static IEnumerable<T> Memoize<T>(this IEnumerable<T> source)
         {
             if (source == null)

Core/Registry/InstalledModule.cs

@@ -146,10 +146,15 @@ private InstalledModule()
         [OnDeserialized]
         private void DeSerialisationFixes(StreamingContext context)
         {
+            if (installed_files == null)
+            {
+                installed_files = new Dictionary<string, InstalledModuleFile>();
+            }
             if (Platform.IsWindows)
             {
                 // We need case insensitive path matching on Windows
-                installed_files = new Dictionary<string, InstalledModuleFile>(installed_files, StringComparer.OrdinalIgnoreCase);
+                installed_files = new Dictionary<string, InstalledModuleFile>(installed_files,
+                                                                              StringComparer.OrdinalIgnoreCase);
             }
         }
 

Core/Registry/Registry.cs

@@ -42,6 +42,7 @@ public class Registry : IEnlistmentNotification, IRegistryQuerier
         private Dictionary<string, string> installed_dlls;
 
         [JsonProperty]
+        [JsonConverter(typeof(JsonParallelDictionaryConverter<InstalledModule>))]
         private Dictionary<string, InstalledModule> installed_modules;
 
         // filename (case insensitive on Windows) => module

Core/Repositories/ProgressImmediate.cs

@@ -0,0 +1,19 @@
+using System;
+
+namespace CKAN
+{
+    public class ProgressImmediate<T> : IProgress<T>
+    {
+        public ProgressImmediate(Action<T> action)
+        {
+            this.action = action;
+        }
+
+        public void Report(T val)
+        {
+            action(val);
+        }
+
+        private readonly Action<T> action;
+    }
+}

Core/Repositories/ProgressScalePercentsByFileSize.cs

@@ -0,0 +1,67 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace CKAN
+{
+    /// <summary>
+    /// Accepts progress updates in terms of percentage of one file within a group
+    /// and translates them into percentages across the whole operation.
+    /// </summary>
+    public class ProgressScalePercentsByFileSizes : IProgress<int>
+    {
+        /// <summary>
+        /// Initialize an percent-to-scaled-percent progress translator
+        /// </summary>
+        /// <param name="percentProgress">The upstream progress object expecting percentages</param>
+        /// <param name="sizes">Sequence of sizes of files in our group</param>
+        public ProgressScalePercentsByFileSizes(IProgress<int> percentProgress,
+                                                IEnumerable<long> sizes)
+        {
+            this.percentProgress = percentProgress;
+            this.sizes           = sizes.ToArray();
+            totalSize            = this.sizes.Sum();
+        }
+
+        /// <summary>
+        /// The IProgress member called when we advance within the current file
+        /// </summary>
+        /// <param name="currentFilePercent">How far into the current file we are</param>
+        public void Report(int currentFilePercent)
+        {
+            if (basePercent < 100 && currentIndex < sizes.Length)
+            {
+                var percent = basePercent + (int)(currentFilePercent * sizes[currentIndex] / totalSize);
+                // Only report each percentage once, to avoid spamming UI calls
+                if (percent > lastPercent)
+                {
+                    percentProgress.Report(percent);
+                    lastPercent = percent;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Call this when you move on from one file to the next
+        /// </summary>
+        public void NextFile()
+        {
+            doneSize += sizes[currentIndex];
+            basePercent = (int)(100 * doneSize / totalSize);
+            ++currentIndex;
+            if (basePercent > lastPercent)
+            {
+                percentProgress.Report(basePercent);
+                lastPercent = basePercent;
+            }
+        }
+
+        private IProgress<int> percentProgress;
+        private long[]         sizes;
+        private long           totalSize;
+        private long           doneSize     = 0;
+        private int            currentIndex = 0;
+        private int            basePercent  = 0;
+        private int            lastPercent  = -1;
+    }
+}

Core/Repositories/ReadProgressStream.cs

@@ -20,11 +20,14 @@ public ReadProgressStream(Stream stream, IProgress<long> progress)
         public override int Read(byte[] buffer, int offset, int count)
         {
             int amountRead = base.Read(buffer, offset, count);
-            long newProgress = Position;
-            if (newProgress > lastProgress)
+            if (progress != null)
             {
-                progress?.Report(newProgress);
-                lastProgress = newProgress;
+                long newProgress = Position;
+                if (newProgress > lastProgress)
+                {
+                    progress?.Report(newProgress);
+                    lastProgress = newProgress;
+                }
             }
             return amountRead;
         }

Core/Repositories/RepositoryData.cs

@@ -3,6 +3,8 @@
 using System.Text;
 using System.Linq;
 using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Runtime.Serialization;
 
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -26,6 +28,7 @@ public class RepositoryData
         /// The available modules from this repository
         /// </summary>
         [JsonProperty("available_modules", NullValueHandling = NullValueHandling.Ignore)]
+        [JsonConverter(typeof(JsonParallelDictionaryConverter<AvailableModule>))]
         public readonly Dictionary<string, AvailableModule> AvailableModules;
 
         /// <summary>
@@ -92,19 +95,37 @@ public void SaveTo(string path)
         /// Load a previously cached repo data object from JSON on disk
         /// </summary>
         /// <param name="path">Filename of the JSON file to load</param>
+        /// <param name="progress">Progress notifier to receive updates of percent completion of this file</param>
         /// <returns>A repo data object or null if loading fails</returns>
-        public static RepositoryData FromJson(string path, IProgress<long> progress)
+        public static RepositoryData FromJson(string path, IProgress<int> progress)
         {
             try
             {
+                long fileSize = new FileInfo(path).Length;
                 log.DebugFormat("Trying to load repository data from {0}", path);
                 // Ain't OOP grand?!
                 using (var stream = File.Open(path, FileMode.Open))
-                using (var progressStream = new ReadProgressStream(stream, progress))
+                using (var progressStream = new ReadProgressStream(
+                    stream,
+                    progress == null
+                        ? null
+                        // Treat JSON parsing as the first 50%
+                        : new ProgressImmediate<long>(p => progress.Report((int)(50 * p / fileSize)))))
                 using (var reader = new StreamReader(progressStream))
                 using (var jStream = new JsonTextReader(reader))
                 {
-                    return new JsonSerializer().Deserialize<RepositoryData>(jStream);
+                    var settings = new JsonSerializerSettings()
+                    {
+                        Context = new StreamingContext(
+                            StreamingContextStates.Other,
+                            progress == null
+                                ? null
+                                : new ProgressImmediate<int>(p =>
+                                    // Treat CkanModule creation as the last 50%
+                                    progress.Report(50 + p / 2))),
+                    };
+                    return JsonSerializer.Create(settings)
+                                         .Deserialize<RepositoryData>(jStream);
                 }
             }
             catch (Exception exc)

Core/Repositories/RepositoryDataManager.cs

@@ -100,7 +100,7 @@ public void Prepopulate(List<Repository> repos, IProgress<int> percentProgress)
                                                                                   new FileInfo(tuple.Item2).Length))
                                      .ToList();
             // Translate from file group offsets to percent
-            var progress = new ProgressFilesOffsetsToPercent(
+            var progress = new ProgressScalePercentsByFileSizes(
                 percentProgress, reposAndSizes.Select(tuple => tuple.Item2));
             foreach (var repo in reposAndSizes.Select(tuple => tuple.Item1))
             {
@@ -258,10 +258,11 @@ private RepositoryData GetRepoData(Repository repo)
                 ? data
                 : LoadRepoData(repo, null);
 
-        private RepositoryData LoadRepoData(Repository repo, IProgress<long> progress)
+        private RepositoryData LoadRepoData(Repository repo, IProgress<int> progress)
         {
-            log.DebugFormat("Looking for data in {0}", GetRepoDataPath(repo));
-            var data = RepositoryData.FromJson(GetRepoDataPath(repo), progress);
+            var path = GetRepoDataPath(repo);
+            log.DebugFormat("Looking for data in {0}", path);
+            var data = RepositoryData.FromJson(path, progress);
             if (data != null)
             {
                 log.Debug("Found it! Adding...");

Core/Types/CkanModule.cs

@@ -6,6 +6,7 @@
 using System.Runtime.Serialization;
 using System.Text;
 using System.Text.RegularExpressions;
+using System.Reflection;
 
 using Autofac;
 using log4net;
@@ -281,6 +282,11 @@ internal CkanModule()
             // We don't have this passed in, so we'll ask the service locator
             // directly. Yuck.
             _comparator = ServiceLocator.Container.Resolve<IGameComparator>();
+            download_content_type = typeof(CkanModule).GetTypeInfo()
+                                                      .GetDeclaredField("download_content_type")
+                                                      .GetCustomAttribute<DefaultValueAttribute>()
+                                                      .Value
+                                                      .ToString();
         }
 
         /// <summary>

Core/Types/ModuleInstallDescriptor.cs

@@ -7,6 +7,7 @@
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Runtime.CompilerServices;
+using System.Reflection;
 
 using ICSharpCode.SharpZipLib.Zip;
 using Newtonsoft.Json;
@@ -109,6 +110,11 @@ internal void DeSerialisationFixes(StreamingContext like_i_could_care)
         [JsonConstructor]
         private ModuleInstallDescriptor()
         {
+            install_to = typeof(ModuleInstallDescriptor).GetTypeInfo()
+                                                        .GetDeclaredField("install_to")
+                                                        .GetCustomAttribute<DefaultValueAttribute>()
+                                                        .Value
+                                                        .ToString();
         }
 
         /// <summary>