Parallelize for performance, relationship resolver improvements

Cmdline/Action/Import.cs

@@ -45,18 +45,17 @@ public int RunCommand(CKAN.GameInstance instance, object options)
                 else
                 {
                     log.InfoFormat("Importing {0} files", toImport.Count);
-                    var toInstall = new List<string>();
+                    var toInstall = new List<CkanModule>();
                     var installer = new ModuleInstaller(instance, manager.Cache, user);
                     var regMgr    = RegistryManager.Instance(instance, repoData);
-                    installer.ImportFiles(toImport, user, mod => toInstall.Add(mod.identifier), regMgr.registry, !opts.Headless);
+                    installer.ImportFiles(toImport, user, mod => toInstall.Add(mod), regMgr.registry, !opts.Headless);
                     HashSet<string> possibleConfigOnlyDirs = null;
                     if (toInstall.Count > 0)
                     {
-                        installer.InstallList(
-                            toInstall,
-                            new RelationshipResolverOptions(),
-                            regMgr,
-                            ref possibleConfigOnlyDirs);
+                        installer.InstallList(toInstall,
+                                              new RelationshipResolverOptions(),
+                                              regMgr,
+                                              ref possibleConfigOnlyDirs);
                     }
                     return Exit.OK;
                 }

Cmdline/Action/Install.cs

@@ -136,7 +136,15 @@ public int RunCommand(CKAN.GameInstance instance, object raw_options)
                 {
                     HashSet<string> possibleConfigOnlyDirs = null;
                     var installer = new ModuleInstaller(instance, manager.Cache, user);
-                    installer.InstallList(modules, install_ops, regMgr, ref possibleConfigOnlyDirs);
+                    installer.InstallList(modules.Select(arg => CkanModule.FromIDandVersion(
+                                                                    regMgr.registry, arg,
+                                                                    options.allow_incompatible
+                                                                        ? null
+                                                                        : instance.VersionCriteria()))
+                                                 .ToList(),
+                                          install_ops,
+                                          regMgr,
+                                          ref possibleConfigOnlyDirs);
                     user.RaiseMessage("");
                     done = true;
                 }
@@ -213,8 +221,7 @@ public int RunCommand(CKAN.GameInstance instance, object raw_options)
                 }
                 catch (InconsistentKraken ex)
                 {
-                    // The prettiest Kraken formats itself for us.
-                    user.RaiseError("{0}", ex.InconsistenciesPretty);
+                    user.RaiseError("{0}", ex.Message);
                     user.RaiseMessage(Properties.Resources.InstallCancelled);
                     return Exit.ERROR;
                 }

Cmdline/Action/List.cs

@@ -1,4 +1,5 @@
 using System;
+using System.IO;
 using System.Collections.Generic;
 
 using log4net;
@@ -39,7 +40,9 @@ public int RunCommand(CKAN.GameInstance instance, object raw_options)
             if (!(options.porcelain) && exportFileType == null)
             {
                 user.RaiseMessage("");
-                user.RaiseMessage(Properties.Resources.ListGameFound, instance.game.ShortName, instance.GameDir());
+                user.RaiseMessage(Properties.Resources.ListGameFound,
+                                  instance.game.ShortName,
+                                  instance.GameDir().Replace('/', Path.DirectorySeparatorChar));
                 user.RaiseMessage("");
                 user.RaiseMessage(Properties.Resources.ListGameVersion, instance.game.ShortName, instance.Version());
                 user.RaiseMessage("");

Cmdline/Action/Upgrade.cs

@@ -134,7 +134,7 @@ public int RunCommand(CKAN.GameInstance instance, object raw_options)
             }
             catch (ModuleNotFoundKraken kraken)
             {
-                user.RaiseMessage(Properties.Resources.UpgradeNotFound, kraken.module);
+                user.RaiseMessage(Properties.Resources.UpgradeNotFound, $"{kraken.module} {kraken.version}");
                 return Exit.ERROR;
             }
             catch (InconsistentKraken kraken)
@@ -166,7 +166,7 @@ public int RunCommand(CKAN.GameInstance instance, object raw_options)
         /// <param name="user">IUser object for output</param>
         /// <param name="instance">Game instance to use</param>
         /// <param name="modules">List of modules to upgrade</param>
-        public void UpgradeModules(GameInstanceManager manager, IUser user, CKAN.GameInstance instance, bool ConfirmPrompt, List<CkanModule> modules)
+        private void UpgradeModules(GameInstanceManager manager, IUser user, CKAN.GameInstance instance, bool ConfirmPrompt, List<CkanModule> modules)
         {
             UpgradeModules(manager, user, instance, repoData,
                 (ModuleInstaller installer, NetAsyncModulesDownloader downloader, RegistryManager regMgr, ref HashSet<string> possibleConfigOnlyDirs) =>
@@ -183,12 +183,19 @@ public void UpgradeModules(GameInstanceManager manager, IUser user, CKAN.GameIns
         /// <param name="user">IUser object for output</param>
         /// <param name="instance">Game instance to use</param>
         /// <param name="identsAndVersions">List of identifier[=version] to upgrade</param>
-        public void UpgradeModules(GameInstanceManager manager, IUser user, CKAN.GameInstance instance, List<string> identsAndVersions)
+        private void UpgradeModules(GameInstanceManager manager, IUser user, CKAN.GameInstance instance, List<string> identsAndVersions)
         {
             UpgradeModules(manager, user, instance, repoData,
                 (ModuleInstaller installer, NetAsyncModulesDownloader downloader, RegistryManager regMgr, ref HashSet<string> possibleConfigOnlyDirs) =>
-                    installer.Upgrade(identsAndVersions, downloader,
-                        ref possibleConfigOnlyDirs, regMgr, true),
+                    installer.Upgrade(
+                        identsAndVersions.Select(arg => CkanModule.FromIDandVersion(
+                                                            regMgr.registry, arg,
+                                                            instance.VersionCriteria()))
+                                         .ToList(),
+                        downloader,
+                        ref possibleConfigOnlyDirs,
+                        regMgr,
+                        true),
                 m => identsAndVersions.Add(m.identifier)
             );
         }

Cmdline/Main.cs

@@ -323,7 +323,7 @@ private static int Scan(CKAN.GameInstance inst, IUser user, string next_command
 
                 if (next_command == null)
                 {
-                    user.RaiseError("{0}", kraken.InconsistenciesPretty);
+                    user.RaiseError("{0}", kraken.Message);
                     user.RaiseError(Properties.Resources.ScanNotSaved);
                 }
                 else

ConsoleUI/GameInstanceListScreen.cs

@@ -2,6 +2,8 @@
 using System.IO;
 using System.Collections.Generic;
 using System.ComponentModel;
+using System.Linq;
+
 using CKAN.Versioning;
 using CKAN.ConsoleUI.Toolkit;
 
@@ -75,7 +77,9 @@ public GameInstanceListScreen(GameInstanceManager mgr, RepositoryDataManager rep
                     new List<string>()
                 );
 
-                if (TryGetInstance(theme, instanceList.Selection, repoData, (ConsoleTheme th) => { d.Run(th, (ConsoleTheme thm) => {}); })) {
+                if (TryGetInstance(theme, instanceList.Selection, repoData,
+                                   (ConsoleTheme th) => { d.Run(th, (ConsoleTheme thm) => {}); },
+                                   null)) {
                     try {
                         manager.SetCurrentInstance(instanceList.Selection.Name);
                     } catch (Exception ex) {
@@ -108,7 +112,9 @@ public GameInstanceListScreen(GameInstanceManager mgr, RepositoryDataManager rep
                     string.Format(Properties.Resources.InstanceListLoadingInstance, instanceList.Selection.Name),
                     new List<string>()
                 );
-                TryGetInstance(theme, instanceList.Selection, repoData, (ConsoleTheme th) => { d.Run(theme, (ConsoleTheme thm) => {}); });
+                TryGetInstance(theme, instanceList.Selection, repoData,
+                               (ConsoleTheme th) => { d.Run(theme, (ConsoleTheme thm) => {}); },
+                               null);
                 // Still launch the screen even if the load fails,
                 // because you need to be able to fix the name/path.
                 LaunchSubScreen(theme, new GameInstanceEditScreen(manager, repoData, instanceList.Selection));
@@ -170,10 +176,15 @@ protected override string MenuTip()
         /// <param name="ksp">Game instance</param>
         /// <param name="repoData">Repository data manager providing info from repos</param>
         /// <param name="render">Function that shows a loading message</param>
+        /// <param name="progress">Function to call with progress updates 0-100</param>
         /// <returns>
         /// True if successfully loaded, false if it's locked or the registry was corrupted, etc.
         /// </returns>
-        public static bool TryGetInstance(ConsoleTheme theme, GameInstance ksp, RepositoryDataManager repoData, Action<ConsoleTheme> render)
+        public static bool TryGetInstance(ConsoleTheme          theme,
+                                          GameInstance          ksp,
+                                          RepositoryDataManager repoData,
+                                          Action<ConsoleTheme>  render,
+                                          IProgress<int>        progress)
         {
             bool retry;
             do {
@@ -183,7 +194,10 @@ public static bool TryGetInstance(ConsoleTheme theme, GameInstance ksp, Reposito
                     // Show loading message
                     render(theme);
                     // Try to get the lock; this will throw if another instance is in there
-                    RegistryManager.Instance(ksp, repoData);
+                    var regMgr = RegistryManager.Instance(ksp, repoData);
+                    repoData.Prepopulate(regMgr.registry.Repositories.Values.ToList(),
+                                         progress);
+                    var compat = regMgr.registry.CompatibleModules(ksp.VersionCriteria());
 
                 } catch (RegistryInUseKraken k) {
 
@@ -215,7 +229,9 @@ public static bool TryGetInstance(ConsoleTheme theme, GameInstance ksp, Reposito
                 } catch (Exception e) {
 
                     ConsoleMessageDialog errd = new ConsoleMessageDialog(
-                        string.Format(Properties.Resources.InstanceListLoadingError, Path.Combine(ksp.CkanDir(), "registry.json"), e.Message),
+                        string.Format(Properties.Resources.InstanceListLoadingError,
+                                      Path.Combine(ksp.CkanDir(), "registry.json").Replace('/', Path.DirectorySeparatorChar),
+                                      e.ToString()),
                         new List<string>() { Properties.Resources.OK }
                     );
                     errd.Run(theme);

ConsoleUI/InstallScreen.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Transactions;
 using System.Collections.Generic;
+using System.Linq;
 
 using CKAN.ConsoleUI.Toolkit;
 
@@ -76,7 +77,11 @@ public override void Run(ConsoleTheme theme, Action<ConsoleTheme> process = null
                             plan.Install.Clear();
                         }
                         if (plan.Upgrade.Count > 0) {
-                            inst.Upgrade(plan.Upgrade, dl, ref possibleConfigOnlyDirs, regMgr);
+                            inst.Upgrade(plan.Upgrade
+                                             .Select(ident => regMgr.registry.LatestAvailable(
+                                                                  ident, manager.CurrentInstance.VersionCriteria()))
+                                             .ToList(),
+                                         dl, ref possibleConfigOnlyDirs, regMgr);
                             plan.Upgrade.Clear();
                         }
                         if (plan.Replace.Count > 0) {
@@ -116,7 +121,7 @@ public override void Run(ConsoleTheme theme, Action<ConsoleTheme> process = null
                     } catch (MissingCertificateKraken ex) {
                         RaiseError(ex.ToString());
                     } catch (InconsistentKraken ex) {
-                        RaiseError(ex.InconsistenciesPretty);
+                        RaiseError(ex.Message);
                     } catch (TooManyModsProvideKraken ex) {
 
                         ConsoleChoiceDialog<CkanModule> ch = new ConsoleChoiceDialog<CkanModule>(

ConsoleUI/ModListScreen.cs

@@ -483,7 +483,7 @@ private bool ScanForMods()
                 regMgr.ScanUnmanagedFiles();
             } catch (InconsistentKraken ex) {
                 // Warn about inconsistent state
-                RaiseError(Properties.Resources.ModListScanBad, ex.InconsistenciesPretty);
+                RaiseError(Properties.Resources.ModListScanBad, ex.Message);
             }
             return true;
         }

ConsoleUI/SplashScreen.cs

@@ -28,7 +28,10 @@ public bool Run(ConsoleTheme theme)
         {
             // If there's a default instance, try to get the lock for it.
             GameInstance ksp = manager.CurrentInstance ?? manager.GetPreferredInstance();
-            if (ksp != null && !GameInstanceListScreen.TryGetInstance(theme, ksp, repoData, (ConsoleTheme th) => Draw(th, false))) {
+            if (ksp != null
+                && !GameInstanceListScreen.TryGetInstance(theme, ksp, repoData,
+                                                          (ConsoleTheme th) => Draw(th, false),
+                                                          new Progress<int>(p => drawProgressBar(theme, 22, 20, p)))) {
                 Console.ResetColor();
                 Console.Clear();
                 Console.CursorVisible = true;
@@ -94,6 +97,35 @@ private void Draw(ConsoleTheme theme, bool pressAny = false)
             }
         }
 
+        private void drawProgressBar(ConsoleTheme theme, int y, int w, int percent)
+        {
+            lock (progBarMutex)
+            {
+                try {
+                    var doubleWidth = percent * (w - 2) / 50;
+                    if (doubleWidth > lastProgDblW)
+                    {
+                        int lp = (Console.WindowWidth - w) / 2;
+                        var bar = new string(Symbols.fullBlock, percent * (w - 2) / 100);
+                        if ((doubleWidth & 1) == 1)
+                        {
+                            // "Cheat" an extra half character to make the bar more precise
+                            bar += Symbols.leftHalfBlock;
+                        }
+                        // This can throw if the screen is too small
+                        Console.SetCursorPosition(lp, y);
+                        Console.ForegroundColor = theme.SplashNormalFg;
+                        Console.Write("[");
+                        Console.ForegroundColor = theme.SplashAccentFg;
+                        Console.Write(bar.PadRight(w - 2, ' '));
+                        Console.ForegroundColor = theme.SplashNormalFg;
+                        Console.Write("]");
+                        lastProgDblW = doubleWidth;
+                    }
+                } catch { }
+            }
+        }
+
         private void drawCentered(int y, string val)
         {
             int lp = (Console.WindowWidth - val.Length) / 2;
@@ -106,6 +138,8 @@ private void drawCentered(int y, string val)
 
         private GameInstanceManager   manager;
         private RepositoryDataManager repoData;
+        private int                   lastProgDblW = -1;
+        private object                progBarMutex = new object();
     }
 
 }

ConsoleUI/Toolkit/Symbols.cs

@@ -128,6 +128,14 @@ public static class Symbols {
         /// and the letters in the splash screen ASCII art
         /// </summary>
         public static readonly char lowerHalfBlock = cp437c(0xdc);
+        /// <summary>
+        /// Full block used for most of a progress bar
+        /// </summary>
+        public static readonly char fullBlock = cp437c(0xdb);
+        /// <summary>
+        /// Left half block used for right edge of odd width progress bar
+        /// </summary>
+        public static readonly char leftHalfBlock = cp437c(0xdd);
 
         private static char   cp437c(byte num) { return dosCodePage.GetChars(new byte[] {num})[0]; }
         private static string cp437s(byte num) { return $"{cp437c(num)}";                          }

Core/Converters/JsonParallelDictionaryConverter.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Linq;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+using CKAN.Extensions;
+
+namespace CKAN
+{
+    /// <summary>
+    /// A converter that loads a dictionary in parallel,
+    /// use with large collections of complex objects for a speed boost
+    /// </summary>
+    public class JsonParallelDictionaryConverter<V> : JsonConverter
+    {
+        public override object ReadJson(JsonReader     reader,
+                                        Type           objectType,
+                                        object         existingValue,
+                                        JsonSerializer serializer)
+            => ParseWithProgress(JObject.Load(reader)
+                                        .Properties()
+                                        .ToArray(),
+                                 serializer);
+
+        private object ParseWithProgress(JProperty[]    properties,
+                                         JsonSerializer serializer)
+            => Partitioner.Create(properties, true)
+                          .AsParallel()
+                          .WithMergeOptions(ParallelMergeOptions.NotBuffered)
+                          .Select(prop => new KeyValuePair<string, V>(
+                                              prop.Name,
+                                              prop.Value.ToObject<V>()))
+                          .WithProgress(properties.Length,
+                                        serializer.Context.Context as IProgress<int>)
+                          .ToDictionary();
+
+        public override bool CanWrite => false;
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            throw new NotImplementedException();
+        }
+
+        // Only convert when we're an explicit attribute
+        public override bool CanConvert(Type object_type) => false;
+    }
+}