Allow locking nim version in nimble.lock

- Fixes #953

Allow having nim as locked dependency.

- I will add unit tests once we agree on the approach and once nimble related
changes in nim are merged (I will link the PR in comment). Ditto for the
documentation. In order that change to work we have to add nim package in nimble
packages repo and also add alias compiler -> nim to avoid breaking backward
compatibility.

Here it is the flow:

``` bash
nimble develop nim
nimble lock
```

After that `nimble install` and `nimble build` commands will use the locked
`nim` version

src/nimble.nim

@@ -67,7 +67,7 @@ proc initPkgList(pkgInfo: PackageInfo, options: Options): seq[PackageInfo] =
 proc install(packages: seq[PkgTuple], options: Options,
              doPrompt, first, fromLockFile: bool): PackageDependenciesInfo
 
-proc processFreeDependencies(pkgInfo: PackageInfo, options: Options):
+proc processFreeDependencies(pkgInfo: PackageInfo, options: Options, nimAsDependency = false):
     HashSet[PackageInfo] =
   ## Verifies and installs dependencies.
   ##
@@ -237,7 +237,7 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string],
     # `quoteShell` would be more robust than `\"` (and avoid quoting when
     # un-necessary) but would require changing `extractBin`
     let cmd = "$# $# --colors:on --noNimblePath $# $# $#" % [
-      getNimBin(options).quoteShell, pkgInfo.backend, join(args, " "),
+      pkgInfo.getNimBin(options).quoteShell, pkgInfo.backend, join(args, " "),
       outputOpt, input.quoteShell]
     try:
       doCmd(cmd)
@@ -343,7 +343,7 @@ proc packageExists(pkgInfo: PackageInfo, options: Options):
     fillMetaData(oldPkgInfo, pkgDestDir, true)
     return some(oldPkgInfo)
 
-proc processLockedDependencies(pkgInfo: PackageInfo, options: Options):
+proc processLockedDependencies(pkgInfo: PackageInfo, options: Options, onlyNim = false):
   HashSet[PackageInfo]
 
 proc processAllDependencies(pkgInfo: PackageInfo, options: Options):
@@ -353,6 +353,22 @@ proc processAllDependencies(pkgInfo: PackageInfo, options: Options):
   else:
     pkgInfo.processFreeDependencies(options)
 
+proc useLockedNimIfNeeded(pkgInfo: PackageInfo, options: var Options) =
+  if pkgInfo.lockedDeps.len > 0:
+    var deps = pkgInfo.processLockedDependencies(options, true)
+    if deps.len != 0:
+      # process the first entry (hash.pop is triggering warnings)
+      for nimDep in deps:
+        const binaryName = when defined(windows): "nim.exe" else: "nim"
+        let nim = nimDep.getRealDir() / "bin" / binaryName
+
+        if not fileExists(nim):
+          raise nimbleError("Trying to use nim from $1 " % nimDep.getRealDir(),
+                            "If you are using develop mode nim make sure to compile it.")
+
+        options.nim = nim
+        display("Info:", "using $1 for compilation" % options.nim, priority = HighPriority)
+
 proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
                     url: string, first: bool, fromLockFile: bool,
                     vcsRevision = notSetSha1Hash):
@@ -424,10 +440,14 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
     result.pkg = oldPkg
     return
 
+  # nim is intended only for local project local usage, so avoid installing it
+  # in .nimble/bin
+  let isNimPackage = pkgInfo.basicInfo.name.isNim
+
   # Build before removing an existing package (if one exists). This way
   # if the build fails then the old package will still be installed.
 
-  if pkgInfo.bin.len > 0:
+  if pkgInfo.bin.len > 0 and not isNimPackage:
     let paths = result.deps.map(dep => dep.getRealDir())
     let flags = if options.action.typ in {actionInstall, actionPath, actionUninstall, actionDevelop}:
                   options.action.passNimFlags
@@ -465,7 +485,7 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
     filesInstalled.incl copyFileD(pkgInfo.myPath, dest)
 
     var binariesInstalled: HashSet[string]
-    if pkgInfo.bin.len > 0:
+    if pkgInfo.bin.len > 0 and not pkgInfo.basicInfo.name.isNim:
       # Make sure ~/.nimble/bin directory is created.
       createDir(binDir)
       # Set file permissions to +x for all binaries built,
@@ -625,7 +645,7 @@ proc installDependency(pkgInfo: PackageInfo, downloadInfo: DownloadInfo,
 
   return newlyInstalledPkgInfo
 
-proc processLockedDependencies(pkgInfo: PackageInfo, options: Options):
+proc processLockedDependencies(pkgInfo: PackageInfo, options: Options, onlyNim = false):
     HashSet[PackageInfo] =
   # Returns a hash set with `PackageInfo` of all packages from the lock file of
   # the package `pkgInfo` by getting the info for develop mode dependencies from
@@ -745,9 +765,10 @@ proc build(pkgInfo: PackageInfo, options: Options) =
   var args = options.getCompilationFlags()
   buildFromDir(pkgInfo, paths, args, options)
 
-proc build(options: Options) =
+proc build(options: var Options) =
   let dir = getCurrentDir()
   let pkgInfo = getPkgInfo(dir, options)
+  useLockedNimIfNeeded(pkgInfo, options)
   pkgInfo.build(options)
 
 proc clean(options: Options) =
@@ -802,7 +823,7 @@ proc execBackend(pkgInfo: PackageInfo, options: Options) =
             "backend") % [bin, pkgInfo.basicInfo.name, backend], priority = HighPriority)
 
   doCmd("$# $# --noNimblePath $# $# $#" %
-        [getNimBin(options).quoteShell,
+        [pkgInfo.getNimBin(options).quoteShell,
          backend,
          join(args, " "),
          bin.quoteShell,
@@ -1334,7 +1355,7 @@ proc developFreeDependencies(pkgInfo: PackageInfo,
          "developFreeDependencies needs pkgInfo.requires"
 
   for dep in pkgInfo.requires:
-    if dep.name == "nimrod" or dep.name == "nim":
+    if dep.name.isNim:
       continue
 
     let resolvedDep = dep.resolveAlias(options)
@@ -1636,8 +1657,14 @@ proc lock(options: Options) =
 
   var errors = validateDevModeDepsWorkingCopiesBeforeLock(pkgInfo, options)
 
-  let dependencies = pkgInfo.processFreeDependencies(options).map(
-    pkg => pkg.toFullInfo(options)).toSeq
+  let
+    includeNim =
+      pkgInfo.lockedDeps.contains("compiler") or
+      pkgInfo.getDevelopDependencies(options).contains("nim")
+    dependencies = pkgInfo.processFreeDependencies(options, includeNim)
+      .map(pkg => pkg.toFullInfo(options))
+      .toSeq
+
   pkgInfo.validateDevelopDependenciesVersionRanges(dependencies, options)
   var dependencyGraph = buildDependencyGraph(dependencies, options)
 
@@ -1984,6 +2011,10 @@ proc doAction(options: var Options) =
   of actionRefresh:
     refresh(options)
   of actionInstall:
+    if options.action.packages.len == 0:
+      let pkgInfo = getPkgInfo(getCurrentDir(), options)
+      useLockedNimIfNeeded(pkgInfo, options)
+
     let (_, pkgInfo) = install(options.action.packages, options,
                                doPrompt = true,
                                first = true,

src/nimblepkg/options.nim

@@ -435,6 +435,20 @@ proc setNimBin*(options: var Options) =
       raise nimbleError(
         "Unable to find `nim` binary - add to $PATH or use `--nim`")
 
+proc getNimbleFileDir*(pkgInfo: PackageInfo): string =
+  pkgInfo.myPath.splitFile.dir
+
+proc getNimBin*(pkgInfo: PackageInfo, options: Options): string =
+  if pkgInfo.basicInfo.name == "nim":
+    let binaryPath =  when defined(windows):
+        "bin\nim.exe"
+      else:
+        "bin/nim"
+    result = pkgInfo.getNimbleFileDir() / binaryPath
+    display("Info:", "compiling nim package using $1" % result, priority = HighPriority)
+  else:
+    result = options.nim
+
 proc getNimBin*(options: Options): string =
   return options.nim
 

src/nimblepkg/packageinfo.nim

@@ -357,8 +357,6 @@ proc findAllPkgs*(pkglist: seq[PackageInfo], dep: PkgTuple): seq[PackageInfo] =
     if withinRange(pkg, dep.ver):
       result.add pkg
 
-proc getNimbleFileDir*(pkgInfo: PackageInfo): string =
-  pkgInfo.myPath.splitFile.dir
 
 proc getRealDir*(pkgInfo: PackageInfo): string =
   ## Returns the directory containing the package source files.
@@ -537,6 +535,9 @@ proc hash*(x: PackageInfo): Hash =
 proc getNameAndVersion*(pkgInfo: PackageInfo): string =
   &"{pkgInfo.basicInfo.name}@{pkgInfo.basicInfo.version}"
 
+proc isNim*(name: string): bool =
+  result = name == "nim" or name == "nimrod" or name == "compiler"
+
 when isMainModule:
   import unittest
 

src/nimblepkg/packageparser.nim

@@ -184,7 +184,9 @@ proc validatePackageInfo(pkgInfo: PackageInfo, options: Options) =
       raise validationError("'" & pkgInfo.backend &
           "' is an invalid backend.", false)
 
-  validatePackageStructure(pkginfo, options)
+  # nim is used for building the project, thus no need to validate its structure.
+  if not pkgInfo.basicInfo.name.isNim:
+    validatePackageStructure(pkginfo, options)
 
 proc nimScriptHint*(pkgInfo: PackageInfo) =
   if not pkgInfo.isNimScript:
@@ -323,7 +325,7 @@ proc inferInstallRules(pkgInfo: var PackageInfo, options: Options) =
   # installed.)
   let installInstructions =
     pkgInfo.installDirs.len + pkgInfo.installExt.len + pkgInfo.installFiles.len
-  if installInstructions == 0 and pkgInfo.bin.len > 0:
+  if installInstructions == 0 and pkgInfo.bin.len > 0 and pkgInfo.basicInfo.name != "nim":
     pkgInfo.skipExt.add("nim")
 
   # When a package doesn't specify a `srcDir` it's fair to assume that

src/nimblepkg/tools.nim

@@ -62,11 +62,6 @@ proc tryDoCmdEx*(cmd: string): string {.discardable.} =
     raise nimbleError(tryDoCmdExErrorMessage(cmd, output, exitCode))
   return output
 
-proc getNimBin*: string =
-  result = "nim"
-  if findExe("nim") != "": result = findExe("nim")
-  elif findExe("nimrod") != "": result = findExe("nimrod")
-
 proc getNimrodVersion*(options: Options): Version =
   let vOutput = doCmdEx(getNimBin(options).quoteShell & " -v").output
   var matches: array[0..MaxSubpatterns, string]

src/nimblepkg/topologicalsort.nim

@@ -11,7 +11,7 @@ proc getDependencies(packages: seq[PackageInfo], package: PackageInfo,
   ## package. It is needed because some of the names of the packages in the
   ## `requires` clause of a package could be URLs.
   for dep in package.requires:
-    if dep.name == "nim":
+    if dep.name.isNim:
       continue
     var depPkgInfo = initPackageInfo()
     var found = findPkg(packages, dep, depPkgInfo)

tests/nimdep/nimble.lock

@@ -0,0 +1,15 @@
+{
+  "version": 1,
+  "packages": {
+    "nim": {
+      "version": "1.7.1",
+      "vcsRevision": "6bf21e7b3dbf203baf23069c33bd79545681f466",
+      "url": "https://github.com/nim-lang/Nim.git",
+      "downloadMethod": "git",
+      "dependencies": [],
+      "checksums": {
+        "sha1": "673698534e5fd9fd1bface0277766b00ac2988d0"
+      }
+    }
+  }
+}

tests/nimdep/nimdep.nimble

@@ -0,0 +1,12 @@
+# Package
+
+version       = "0.1.0"
+author        = "Ivan Yonchovski"
+description   = "A new awesome nimble package"
+license       = "MIT"
+srcDir        = "src"
+bin           = @["demo"]
+
+# Dependencies
+
+requires  "nim == 1.7.1"

tests/tlockfile.nim

@@ -590,3 +590,24 @@ requires "nim >= 1.5.1"
         writeDevelopFile(developFileName, @[], @[dep1PkgRepoPath, mainPkgOriginRepoPath])
         let (_, exitCode) = execNimbleYes("--debug", "--verbose", "sync")
         check exitCode == QuitSuccess
+
+  test "can generate lock file for nim as dep":
+    cleanUp()
+    cd "nimdep":
+      removeFile "nimble.develop"
+      removeFile "nimble.lock"
+      removeDir "Nim"
+
+      check execNimbleYes("develop", "nim").exitCode == QuitSuccess
+      cd "Nim":
+        let (_, exitCode) = execNimbleYes("-y", "install")
+        check exitCode == QuitSuccess
+
+      # check if the compiler version will be used when doing build
+      testLockFile(@[("nim", "Nim")], isNew = true)
+      removeFile "nimble.develop"
+      removeDir "Nim"
+
+      let (output, exitCodeInstall) = execNimbleYes("-y", "build")
+      check exitCodeInstall == QuitSuccess
+      check output.contains("bin/nim for compilation")