diff --git a/.gitignore b/.gitignore index ec6622fd..99d2cbd4 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ src/nimblepkg/version # Test procedure artifacts *.nims /buildTests +/tests/nimdep/nimble.lock diff --git a/src/nimble.nim b/src/nimble.nim index e175d02c..673411c9 100644 --- a/src/nimble.nim +++ b/src/nimble.nim @@ -27,6 +27,7 @@ const nimbleConfigFileName* = "config.nims" gitIgnoreFileName = ".gitignore" hgIgnoreFileName = ".hgignore" + nimblePathsEnv = "__NIMBLE_PATHS" proc refresh(options: Options) = ## Downloads the package list from the specified URL. @@ -67,7 +68,21 @@ proc initPkgList(pkgInfo: PackageInfo, options: Options): seq[PackageInfo] = proc install(packages: seq[PkgTuple], options: Options, doPrompt, first, fromLockFile: bool): PackageDependenciesInfo -proc processFreeDependencies(pkgInfo: PackageInfo, options: Options): +proc checkSatisfied(options: Options, dependencies: HashSet[PackageInfo]) = + ## Check if two packages of the same name (but different version) are listed + ## in the path. Throws error if it fails + var pkgsInPath: Table[string, Version] + for pkgInfo in dependencies: + let currentVer = pkgInfo.getConcreteVersion(options) + if pkgsInPath.hasKey(pkgInfo.basicInfo.name) and + pkgsInPath[pkgInfo.basicInfo.name] != currentVer: + raise nimbleError( + "Cannot satisfy the dependency on $1 $2 and $1 $3" % + [pkgInfo.basicInfo.name, $currentVer, $pkgsInPath[pkgInfo.basicInfo.name]]) + pkgsInPath[pkgInfo.basicInfo.name] = currentVer + +proc processFreeDependencies(pkgInfo: PackageInfo, requirements: seq[PkgTuple], + options: Options, nimAsDependency = false): HashSet[PackageInfo] = ## Verifies and installs dependencies. ## @@ -75,41 +90,17 @@ proc processFreeDependencies(pkgInfo: PackageInfo, options: Options): ## during build phase. assert not pkgInfo.isMinimal, "processFreeDependencies needs pkgInfo.requires" - # If the folder we are operating in is the same folder that the .nimble - # file is in then we know we are working the main package - let - isMain = options.startDir == pkgInfo.myPath.splitpath().head - task = options.task - var pkgList {.global.}: seq[PackageInfo] = @[] + var pkgList {.global.}: seq[PackageInfo] once: pkgList = initPkgList(pkgInfo, options) - display("Verifying", "dependencies for $1@$2" % [pkgInfo.basicInfo.name, $pkgInfo.basicInfo.version], priority = HighPriority) - var - reverseDependencies: seq[PackageBasicInfo] = @[] - requirements = pkgInfo.requires - - template addTaskRequirements = - ## Adds all task requirements to list of requirements - for task, requires in pkgInfo.taskRequires: - requirements &= requires - - # Check what task level dependencies need to be added - if isMain: - if task in pkgInfo.taskRequires: - # If this is the main file then add its needed requirements for running a task. - requirements &= pkgInfo.taskRequires[task] - elif options.action.typ in {actionLock, actionSync}: - # We only add top level task requirements into lock file - addTaskRequirements() - if options.action.typ == actionDeps: - addTaskRequirements() + var reverseDependencies: seq[PackageBasicInfo] = @[] for dep in requirements: - if dep.name == "nimrod" or dep.name == "nim": + if not nimAsDependency and dep.name.isNim: let nimVer = getNimrodVersion(options) if not withinRange(nimVer, dep.ver): let msg = "Unsatisfied dependency: " & dep.name & " (" & $dep.ver & ")" @@ -152,21 +143,13 @@ proc processFreeDependencies(pkgInfo: PackageInfo, options: Options): displayInfo(pkgDepsAlreadySatisfiedMsg(dep)) result.incl pkg # Process the dependencies of this dependency. - result.incl processFreeDependencies(pkg.toFullInfo(options), options) + let fullInfo = pkg.toFullInfo(options) + result.incl processFreeDependencies(fullInfo, fullInfo.requires, options) + if not pkg.isLink: reverseDependencies.add(pkg.basicInfo) - # Check if two packages of the same name (but different version) are listed - # in the path. - var pkgsInPath: Table[string, Version] - for pkgInfo in result: - let currentVer = pkgInfo.getConcreteVersion(options) - if pkgsInPath.hasKey(pkgInfo.basicInfo.name) and - pkgsInPath[pkgInfo.basicInfo.name] != currentVer: - raise nimbleError( - "Cannot satisfy the dependency on $1 $2 and $1 $3" % - [pkgInfo.basicInfo.name, $currentVer, $pkgsInPath[pkgInfo.basicInfo.name]]) - pkgsInPath[pkgInfo.basicInfo.name] = currentVer + options.checkSatisfied(result) # We add the reverse deps to the JSON file here because we don't want # them added if the above errorenous condition occurs @@ -237,7 +220,7 @@ proc buildFromDir(pkgInfo: PackageInfo, paths: HashSet[string], # `quoteShell` would be more robust than `\"` (and avoid quoting when # un-necessary) but would require changing `extractBin` let cmd = "$# $# --colors:on --noNimblePath $# $# $#" % [ - getNimBin(options).quoteShell, pkgInfo.backend, join(args, " "), + pkgInfo.getNimBin(options).quoteShell, pkgInfo.backend, join(args, " "), outputOpt, input.quoteShell] try: doCmd(cmd) @@ -343,19 +326,52 @@ proc packageExists(pkgInfo: PackageInfo, options: Options): fillMetaData(oldPkgInfo, pkgDestDir, true) return some(oldPkgInfo) -proc processLockedDependencies(pkgInfo: PackageInfo, options: Options): +proc processLockedDependencies(pkgInfo: PackageInfo, options: Options, onlyNim = false): HashSet[PackageInfo] +proc getDependenciesPaths(pkgInfo: PackageInfo, options: Options): + HashSet[string] + proc processAllDependencies(pkgInfo: PackageInfo, options: Options): HashSet[PackageInfo] = - if pkgInfo.lockedDeps.len > 0: - pkgInfo.processLockedDependencies(options) + if pkgInfo.hasLockedDeps(): + result = pkgInfo.processLockedDependencies(options) else: - pkgInfo.processFreeDependencies(options) + result.incl pkgInfo.processFreeDependencies(pkgInfo.requires, options) + if options.task in pkgInfo.taskRequires: + result.incl pkgInfo.processFreeDependencies(pkgInfo.taskRequires[options.task], options) + + putEnv(nimblePathsEnv, result.map(dep => dep.getRealDir()).toSeq().join("|")) + +proc allDependencies(pkgInfo: PackageInfo, options: Options): HashSet[PackageInfo] = + ## Returns all dependencies for a package (Including tasks) + result.incl pkgInfo.processFreeDependencies(pkgInfo.requires, options) + for requires in pkgInfo.taskRequires.values: + result.incl pkgInfo.processFreeDependencies(requires, options) + +proc useLockedNimIfNeeded(pkgInfo: PackageInfo, options: var Options) = + if pkgInfo.lockedDeps.len > 0 and not options.useSystemNim: + var deps = pkgInfo.processLockedDependencies(options, true) + if deps.len != 0: + # process the first entry (hash.pop is triggering warnings) + for nimDep in deps: + const binaryName = when defined(windows): "nim.exe" else: "nim" + let nim = nimDep.getRealDir() / "bin" / binaryName + + if not fileExists(nim): + raise nimbleError("Trying to use nim from $1 " % nimDep.getRealDir(), + "If you are using develop mode nim make sure to compile it.") + + options.nim = nim + let separator = when defined(windows): ";" else: ":" + + putEnv("PATH", nimDep.getRealDir() / "bin" & separator & getEnv("PATH")) + display("Info:", "using $1 for compilation" % options.nim, priority = HighPriority) proc installFromDir(dir: string, requestedVer: VersionRange, options: Options, url: string, first: bool, fromLockFile: bool, - vcsRevision = notSetSha1Hash): + vcsRevision = notSetSha1Hash, + deps: seq[PackageInfo] = @[]): PackageDependenciesInfo = ## Returns where package has been installed to, together with paths ## to the packages this package depends on. @@ -396,10 +412,12 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options, pkgInfo.metaData.specialVersions.incl requestedVer.spe # Dependencies need to be processed before the creation of the pkg dir. - if first and pkgInfo.lockedDeps.len > 0: + if first and pkgInfo.hasLockedDeps(): result.deps = pkgInfo.processLockedDependencies(depsOptions) elif not fromLockFile: - result.deps = pkgInfo.processFreeDependencies(depsOptions) + result.deps = pkgInfo.processFreeDependencies(pkgInfo.requires, depsOptions) + else: + result.deps = deps.toHashSet if options.depsOnly: result.pkg = pkgInfo @@ -424,10 +442,14 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options, result.pkg = oldPkg return + # nim is intended only for local project local usage, so avoid installing it + # in .nimble/bin + let isNimPackage = pkgInfo.basicInfo.name.isNim + # Build before removing an existing package (if one exists). This way # if the build fails then the old package will still be installed. - if pkgInfo.bin.len > 0: + if pkgInfo.bin.len > 0 and not isNimPackage: let paths = result.deps.map(dep => dep.getRealDir()) let flags = if options.action.typ in {actionInstall, actionPath, actionUninstall, actionDevelop}: options.action.passNimFlags @@ -465,7 +487,7 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options, filesInstalled.incl copyFileD(pkgInfo.myPath, dest) var binariesInstalled: HashSet[string] - if pkgInfo.bin.len > 0: + if pkgInfo.bin.len > 0 and not pkgInfo.basicInfo.name.isNim: # Make sure ~/.nimble/bin directory is created. createDir(binDir) # Set file permissions to +x for all binaries built, @@ -604,7 +626,8 @@ proc downloadDependency(name: string, dep: LockFileDep, options: Options): vcsRevision: vcsRevision) proc installDependency(pkgInfo: PackageInfo, downloadInfo: DownloadInfo, - options: Options): PackageInfo = + options: Options, + deps: seq[PackageInfo]): PackageInfo = ## Installs an already downloaded dependency of the package `pkgInfo`. let (_, newlyInstalledPkgInfo) = installFromDir( downloadInfo.downloadDir, @@ -613,19 +636,20 @@ proc installDependency(pkgInfo: PackageInfo, downloadInfo: DownloadInfo, downloadInfo.url, first = false, fromLockFile = true, - downloadInfo.vcsRevision) + downloadInfo.vcsRevision, + deps = deps) downloadInfo.downloadDir.removeDir - + let deps = pkgInfo.lockedDeps[noTask] for depDepName in downloadInfo.dependency.dependencies: - let depDep = pkgInfo.lockedDeps[depDepName] + let depDep = deps[depDepName] let revDep = (name: depDepName, version: depDep.version, checksum: depDep.checksums.sha1) options.nimbleData.addRevDep(revDep, newlyInstalledPkgInfo) return newlyInstalledPkgInfo -proc processLockedDependencies(pkgInfo: PackageInfo, options: Options): +proc processLockedDependencies(pkgInfo: PackageInfo, options: Options, onlyNim = false): HashSet[PackageInfo] = # Returns a hash set with `PackageInfo` of all packages from the lock file of # the package `pkgInfo` by getting the info for develop mode dependencies from @@ -635,27 +659,20 @@ proc processLockedDependencies(pkgInfo: PackageInfo, options: Options): let developModeDeps = getDevelopDependencies(pkgInfo, options) - # Build list of packages allowed for running. - # This is to stop requirements from unrelated tasks - # needing to be downloaded - var allowedPackages: HashSet[string] - for requirement in pkgInfo.requires: - allowedPackages.incl requirement.name - - for requirement in pkgInfo.taskRequires.getOrDefault(options.task): - allowedPackages.incl requirement.name - - for name, dep in pkgInfo.lockedDeps: - if name in allowedPackages: - if developModeDeps.hasKey(name): - result.incl developModeDeps[name][] - elif isInstalled(name, dep, options): - result.incl getDependency(name, dep, options) - elif not options.offline: - let downloadResult = downloadDependency(name, dep, options) - result.incl installDependency(pkgInfo, downloadResult, options) - else: - raise nimbleError("Unsatisfied dependency: " & pkgInfo.basicInfo.name) + for name, dep in pkgInfo.lockedDepsFor(options): + if onlyNim and not name.isNim: + continue + if developModeDeps.hasKey(name): + result.incl developModeDeps[name][] + elif isInstalled(name, dep, options): + result.incl getDependency(name, dep, options) + elif not options.offline: + let + downloadResult = downloadDependency(name, dep, options) + dependencies = result.toSeq.filterIt(dep.dependencies.contains(it.name)) + result.incl installDependency(pkgInfo, downloadResult, options, dependencies) + else: + raise nimbleError("Unsatisfied dependency: " & pkgInfo.basicInfo.name) proc getDownloadInfo*(pv: PkgTuple, options: Options, doPrompt: bool, ignorePackageCache = false): (DownloadMethod, string, @@ -745,9 +762,10 @@ proc build(pkgInfo: PackageInfo, options: Options) = var args = options.getCompilationFlags() buildFromDir(pkgInfo, paths, args, options) -proc build(options: Options) = +proc build(options: var Options) = let dir = getCurrentDir() let pkgInfo = getPkgInfo(dir, options) + useLockedNimIfNeeded(pkgInfo, options) pkgInfo.build(options) proc clean(options: Options) = @@ -802,7 +820,7 @@ proc execBackend(pkgInfo: PackageInfo, options: Options) = "backend") % [bin, pkgInfo.basicInfo.name, backend], priority = HighPriority) doCmd("$# $# --noNimblePath $# $# $#" % - [getNimBin(options).quoteShell, + [pkgInfo.getNimBin(options).quoteShell, backend, join(args, " "), bin.quoteShell, @@ -1312,13 +1330,13 @@ proc installDevelopPackage(pkgTup: PkgTuple, options: var Options): proc developLockedDependencies(pkgInfo: PackageInfo, alreadyDownloaded: var HashSet[string], options: var Options) = ## Downloads for develop the dependencies from the lock file. - - for name, dep in pkgInfo.lockedDeps: - if dep.url.removeTrailingGitString notin alreadyDownloaded: - let downloadResult = downloadDependency(name, dep, options) - alreadyDownloaded.incl downloadResult.url.removeTrailingGitString - options.action.devActions.add( - (datAdd, downloadResult.downloadDir.normalizedPath)) + for task, deps in pkgInfo.lockedDeps: + for name, dep in deps: + if dep.url.removeTrailingGitString notin alreadyDownloaded: + let downloadResult = downloadDependency(name, dep, options) + alreadyDownloaded.incl downloadResult.url.removeTrailingGitString + options.action.devActions.add( + (datAdd, downloadResult.downloadDir.normalizedPath)) proc check(alreadyDownloaded: HashSet[string], dep: PkgTuple, options: Options): bool = @@ -1334,7 +1352,7 @@ proc developFreeDependencies(pkgInfo: PackageInfo, "developFreeDependencies needs pkgInfo.requires" for dep in pkgInfo.requires: - if dep.name == "nimrod" or dep.name == "nim": + if dep.name.isNim: continue let resolvedDep = dep.resolveAlias(options) @@ -1359,7 +1377,7 @@ proc developAllDependencies(pkgInfo: PackageInfo, options: var Options) = var alreadyDownloadedDependencies {.global.}: HashSet[string] alreadyDownloadedDependencies.incl pkgInfo.metaData.url.removeTrailingGitString - if pkgInfo.lockedDeps.len > 0: + if pkgInfo.hasLockedDeps(): pkgInfo.developLockedDependencies(alreadyDownloadedDependencies, options) else: pkgInfo.developFreeDependencies(alreadyDownloadedDependencies, options) @@ -1577,8 +1595,11 @@ proc mergeLockedDependencies*(pkgInfo: PackageInfo, newDeps: LockFileDeps, options: Options): LockFileDeps = ## Updates the lock file data of already generated lock file with the data ## from a new lock operation. + # Copy across the data in the existing lock file + for deps in pkgInfo.lockedDeps.values: + for name, dep in deps: + result[name] = dep - result = pkgInfo.lockedDeps let developDeps = pkgInfo.getDevelopDependencies(options) for name, dep in newDeps: @@ -1624,6 +1645,16 @@ proc displayLockOperationFinish(didLockFileExist: bool) = lockFileIsGeneratedMsg displaySuccess(msg) +proc check(errors: var ValidationErrors, graph: LockFileDeps) = + ## Checks that the dependency graph has no errors + # throw error only for dependencies that are part of the graph + for name, error in common.dup(errors): + if name notin graph: + errors.del name + + if errors.len > 0: + raise validationErrors(errors) + proc lock(options: Options) = ## Generates a lock file for the package in the current directory or updates ## it if it already exists. @@ -1632,24 +1663,53 @@ proc lock(options: Options) = currentDir = getCurrentDir() pkgInfo = getPkgInfo(currentDir, options) currentLockFile = options.lockFile(currentDir) - doesLockFileExist = displayLockOperationStart(currentLockFile) + lockExists = displayLockOperationStart(currentLockFile) var errors = validateDevModeDepsWorkingCopiesBeforeLock(pkgInfo, options) - let dependencies = pkgInfo.processFreeDependencies(options).map( - pkg => pkg.toFullInfo(options)).toSeq - pkgInfo.validateDevelopDependenciesVersionRanges(dependencies, options) - var dependencyGraph = buildDependencyGraph(dependencies, options) - - # throw error only for dependencies that are part of the graph - for name, error in common.dup(errors): - if not dependencyGraph.contains(name): - errors.del name + # We need to process free dependencies for all tasks. + # Then we can store each task as a seperate sub graph. + let + includeNim = + pkgInfo.lockedDeps.contains("compiler") or + pkgInfo.getDevelopDependencies(options).contains("nim") + deps = pkgInfo.processFreeDependencies(pkgInfo.requires, options, includeNim) + var fullDeps = deps # Deps shared by base and tasks - if errors.len > 0: - raise validationErrors(errors) + # We need to seperate the graph into seperate tasks later + var + baseDepNames: HashSet[string] + taskDepNames: Table[string, HashSet[string]] - if currentLockFile.fileExists: + for dep in deps: + baseDepNames.incl dep.name + + + # Add each individual tasks as partial sub graphs + for task, requires in pkgInfo.taskRequires: + let newDeps = pkgInfo.processFreeDependencies(requires, options) + {.push warning[ProveInit]: off.} + # Don't know why this isn't considered proved + let fullInfo = newDeps.toSeq().map(pkg => pkg.toFullInfo(options)) + {.push warning[ProveInit]: on.} + pkgInfo.validateDevelopDependenciesVersionRanges(fullInfo, options) + # Add in the dependencies that are in this task but not in base + taskDepNames[task] = initHashSet[string]() + for dep in newDeps: + fullDeps.incl dep + if dep.name notin baseDepNames: + taskDepNames[task].incl dep.name + # Reset the deps to what they were before hand. + # Stops dependencies in this task overflowing into the next + fullDeps.incl newDeps + # Now build graph for all dependencies + options.checkSatisfied(fullDeps) + let fullInfo = fullDeps.toSeq().map(pkg => pkg.toFullInfo(options)) + pkgInfo.validateDevelopDependenciesVersionRanges(fullInfo, options) + var graph = buildDependencyGraph(fullInfo, options) + errors.check(graph) + + if lockExists: # If we already have a lock file, merge its data with the newly generated # one. # @@ -1658,14 +1718,27 @@ proc lock(options: Options) = # currently Nimble does not check properly for `require` clauses # satisfaction between all packages, but just greedily picks the best # matching version of dependencies for the currently processed package. + graph = mergeLockedDependencies(pkgInfo, graph, options) + + let (topologicalOrder, _) = topologicalSort(graph) + var lockDeps: AllLockFileDeps + # Now we break up tasks into seperate graphs + lockDeps[noTask] = LockFileDeps() + for task in pkgInfo.taskRequires.keys: + lockDeps[task] = LockFileDeps() + + for dep in topologicalOrder: + if dep in baseDepNames: + lockDeps[noTask][dep] = graph[dep] + else: + # Add the dependency for any task that requires it + for task in pkgInfo.taskRequires.keys: + if dep in taskDepNames[task]: + lockDeps[task][dep] = graph[dep] - dependencyGraph = mergeLockedDependencies(pkgInfo, dependencyGraph, options) - - let (topologicalOrder, _) = topologicalSort(dependencyGraph) - - writeLockFile(currentLockFile, dependencyGraph, topologicalOrder) + writeLockFile(currentLockFile, lockDeps) updateSyncFile(pkgInfo, options) - displayLockOperationFinish(doesLockFileExist) + displayLockOperationFinish(lockExists) proc depsTree(options: Options) = @@ -1675,7 +1748,7 @@ proc depsTree(options: Options) = var errors = validateDevModeDepsWorkingCopiesBeforeLock(pkgInfo, options) - let dependencies = pkgInfo.processFreeDependencies(options).map( + let dependencies = pkgInfo.allDependencies(options).map( pkg => pkg.toFullInfo(options)).toSeq pkgInfo.validateDevelopDependenciesVersionRanges(dependencies, options) var dependencyGraph = buildDependencyGraph(dependencies, options) @@ -1702,7 +1775,7 @@ proc syncWorkingCopy(name: string, path: Path, dependentPkg: PackageInfo, displayInfo(&"Syncing working copy of package \"{name}\" at \"{path}\"...") - let lockedDeps = dependentPkg.lockedDeps + let lockedDeps = dependentPkg.lockedDeps[noTask] assert lockedDeps.hasKey(name), &"Package \"{name}\" must be present in the lock file." @@ -1827,7 +1900,7 @@ proc sync(options: Options) = findValidationErrorsOfDevDepsWithLockFile(pkgInfo, options, errors) for name, error in common.dup(errors): - if not pkgInfo.lockedDeps.contains(name): + if not pkgInfo.lockedDeps.hasPackage(name): errors.del name elif error.kind == vekWorkingCopyNeedsSync: if not options.action.listOnly: @@ -1999,6 +2072,10 @@ proc doAction(options: var Options) = of actionRefresh: refresh(options) of actionInstall: + if options.action.packages.len == 0: + let pkgInfo = getPkgInfo(getCurrentDir(), options) + useLockedNimIfNeeded(pkgInfo, options) + let (_, pkgInfo) = install(options.action.packages, options, doPrompt = true, first = true, @@ -2067,6 +2144,7 @@ proc doAction(options: var Options) = discard pkgInfo.processAllDependencies(optsCopy) # If valid task defined in nimscript, run it var execResult: ExecutionResult[bool] + useLockedNimIfNeeded(pkgInfo, optsCopy) if execCustom(nimbleFile, optsCopy, execResult): if execResult.hasTaskRequestedCommand(): var options = execResult.getOptionsForCommand(optsCopy) diff --git a/src/nimblepkg/developfile.nim b/src/nimblepkg/developfile.nim index 505a50fd..c1e94996 100644 --- a/src/nimblepkg/developfile.nim +++ b/src/nimblepkg/developfile.nim @@ -854,7 +854,7 @@ proc workingCopyNeeds*(dependencyPkg, dependentPkg: PackageInfo, ## if any. let - lockFileVcsRev = dependentPkg.lockedDeps.getOrDefault( + lockFileVcsRev = dependentPkg.lockedDeps.getOrDefault("").getOrDefault( dependencyPkg.basicInfo.name, notSetLockFileDep).vcsRevision syncFile = getSyncFile(dependentPkg) syncFileVcsRev = syncFile.getDepVcsRevision(dependencyPkg.basicInfo.name) diff --git a/src/nimblepkg/lockfile.nim b/src/nimblepkg/lockfile.nim index 19941cec..40f61253 100644 --- a/src/nimblepkg/lockfile.nim +++ b/src/nimblepkg/lockfile.nim @@ -9,9 +9,10 @@ type lfjkVersion = "version" lfjkPackages = "packages" lfjkPkgVcsRevision = "vcsRevision" + lfjkTasks = "tasks" const - lockFileVersion = 1 + lockFileVersion = 2 proc initLockFileDep*: LockFileDep = result = LockFileDep( @@ -22,31 +23,35 @@ proc initLockFileDep*: LockFileDep = const notSetLockFileDep* = initLockFileDep() -proc writeLockFile*(fileName: string, packages: LockFileDeps, - topologicallySortedOrder: seq[string]) = +proc writeLockFile*(fileName: string, packages: AllLockFileDeps) = ## Saves lock file on the disk in topologically sorted order of the ## dependencies. - let packagesJsonNode = newJObject() - for packageName in topologicallySortedOrder: - packagesJsonNode.add packageName, %packages[packageName] - let mainJsonNode = %{ $lfjkVersion: %lockFileVersion, - $lfjkPackages: packagesJsonNode - } + $lfjkPackages: %packages[noTask] + } + # Store task graph seperate + mainJsonNode[$lfjkTasks] = newJObject() + for task, deps in packages: + if task != noTask: + mainJsonNode[$lfjkTasks][task] = %deps var s = mainJsonNode.pretty s.add '\n' writeFile(fileName, s) -proc readLockFile*(filePath: string): LockFileDeps = +proc readLockFile*(filePath: string): AllLockFileDeps = {.warning[UnsafeDefault]: off.} {.warning[ProveInit]: off.} - result = parseFile(filePath)[$lfjkPackages].to(result.typeof) + let data = parseFile(filePath) + result[noTask] = data[$lfjkPackages].to(LockFileDeps) + if $lfjkTasks in data: + for task, deps in data[$lfjkTasks]: + result[task] = deps.to(LockFileDeps) {.warning[ProveInit]: on.} {.warning[UnsafeDefault]: on.} -proc getLockedDependencies*(lockFile: string): LockFileDeps = +proc getLockedDependencies*(lockFile: string): AllLockFileDeps = if lockFile.fileExists: result = lockFile.readLockFile diff --git a/src/nimblepkg/nimscriptapi.nim b/src/nimblepkg/nimscriptapi.nim index 2bd34b2e..bca23450 100644 --- a/src/nimblepkg/nimscriptapi.nim +++ b/src/nimblepkg/nimscriptapi.nim @@ -42,6 +42,7 @@ var project = "" success = false retVal = true + nimblePathsEnv = "__NIMBLE_PATHS" proc requires*(deps: varargs[string]) = ## Call this to set the list of requirements of your Nimble @@ -201,7 +202,7 @@ template task*(name: untyped; description: string; body: untyped): untyped = proc `name Task`*() = body nimbleTasks.add (astToStr(name), description) - + if actionName.len == 0 or actionName == "help": success = true elif actionName == astToStr(name).normalize: @@ -238,3 +239,11 @@ proc getPkgDir*(): string = result = projectFile.rsplit(seps={'/', '\\', ':'}, maxsplit=1)[0] proc thisDir*(): string = getPkgDir() + +proc getPaths*(): seq[string] = + ## Returns the paths to the dependencies + return getEnv(nimblePathsEnv).split("|") + +proc getPathsClause*(): string = + ## Returns the paths to the dependencies as consumed by the nim compiler. + return getPaths().mapIt("--path:" & it.quoteShell).join(" ") diff --git a/src/nimblepkg/options.nim b/src/nimblepkg/options.nim index 7b05e56e..5baec4d8 100644 --- a/src/nimblepkg/options.nim +++ b/src/nimblepkg/options.nim @@ -28,6 +28,7 @@ type pkgInfoCache*: TableRef[string, PackageInfo] showHelp*: bool lockFileName*: string + useSystemNim*: bool showVersion*: bool offline*: bool noColor*: bool @@ -44,6 +45,7 @@ type localdeps*: bool # True if project local deps mode developLocaldeps*: bool # True if local deps + nimble develop pkg1 ... disableSslCertCheck*: bool + disableLockFile*: bool enableTarballs*: bool # Enable downloading of packages as tarballs from GitHub. task*: string # Name of the task that is getting ran package*: string @@ -112,7 +114,7 @@ Usage: nimble [nimbleopts] COMMAND [cmdopts] Commands: install [pkgname, ...] Installs a list of packages. [-d, --depsOnly] Only install dependencies. Leave out pkgname - to install deps for a local nimble package. + to install deps for a local nimble package. [-p, --passNim] Forward specified flag to compiler. [--noRebuild] Don't rebuild binaries if they're up-to-date. develop [pkgname, ...] Clones a list of packages for development. @@ -228,6 +230,9 @@ Nimble Options: --noColor Don't colorise output. --noSSLCheck Don't check SSL certificates. --lock-file Override the lock file name. + --noLockFile Ignore the lock file if present. + --use-system-nim Use system nim and ignore nim from the lock + file if any For more information read the Github readme: https://github.com/nim-lang/nimble#readme @@ -435,6 +440,20 @@ proc setNimBin*(options: var Options) = raise nimbleError( "Unable to find `nim` binary - add to $PATH or use `--nim`") +proc getNimbleFileDir*(pkgInfo: PackageInfo): string = + pkgInfo.myPath.splitFile.dir + +proc getNimBin*(pkgInfo: PackageInfo, options: Options): string = + if pkgInfo.basicInfo.name == "nim": + let binaryPath = when defined(windows): + "bin\nim.exe" + else: + "bin/nim" + result = pkgInfo.getNimbleFileDir() / binaryPath + display("Info:", "compiling nim package using $1" % result, priority = HighPriority) + else: + result = options.nim + proc getNimBin*(options: Options): string = return options.nim @@ -515,9 +534,11 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) = of "nim": result.nim = val of "localdeps", "l": result.localdeps = true of "nosslcheck": result.disableSslCertCheck = true + of "nolockfile": result.disableLockFile = true of "tarballs", "t": result.enableTarballs = true of "package", "p": result.package = val of "lock-file": result.lockFileName = val + of "use-system-nim": result.useSystemNim = true else: isGlobalFlag = false var wasFlagHandled = true diff --git a/src/nimblepkg/packageinfo.nim b/src/nimblepkg/packageinfo.nim index 3d4f9863..0e3add69 100644 --- a/src/nimblepkg/packageinfo.nim +++ b/src/nimblepkg/packageinfo.nim @@ -22,7 +22,7 @@ proc isLoaded*(pkgInfo: PackageInfo): bool = return pkgInfo.myPath.len > 0 proc assertIsLoaded*(pkgInfo: PackageInfo) = - assert pkgInfo.isLoaded, "The package info must be loaded." + assert pkgInfo.isLoaded, "The package info must be loaded. " proc areLockedDepsLoaded*(pkgInfo: PackageInfo): bool = pkgInfo.lockedDeps.len > 0 @@ -37,7 +37,8 @@ proc initPackageInfo*(options: Options, filePath: string): PackageInfo = result.myPath = filePath result.basicInfo.name = fileName result.backend = "c" - result.lockedDeps = options.lockFile(fileDir).getLockedDependencies() + if not options.disableLockFile: + result.lockedDeps = options.lockFile(fileDir).getLockedDependencies() proc toValidPackageName*(name: string): string = for c in name: @@ -200,8 +201,11 @@ proc readPackageList(name: string, options: Options, ignorePackageCache = false) # going further. gPackageJson[name] = newJArray() return gPackageJson[name] - gPackageJson[name] = parseFile(options.getNimbleDir() / "packages_" & - name.toLowerAscii() & ".json") + let file = options.getNimbleDir() / "packages_" & name.toLowerAscii() & ".json" + if file.fileExists: + gPackageJson[name] = parseFile(file) + else: + gPackageJson[name] = newJArray() return gPackageJson[name] proc getPackage*(pkg: string, options: Options, resPkg: var Package, ignorePackageCache = false): bool @@ -357,8 +361,6 @@ proc findAllPkgs*(pkglist: seq[PackageInfo], dep: PkgTuple): seq[PackageInfo] = if withinRange(pkg, dep.ver): result.add pkg -proc getNimbleFileDir*(pkgInfo: PackageInfo): string = - pkgInfo.myPath.splitFile.dir proc getRealDir*(pkgInfo: PackageInfo): string = ## Returns the directory containing the package source files. @@ -525,6 +527,26 @@ proc fullRequirements*(pkgInfo: PackageInfo): seq[PkgTuple] = for requirements in pkgInfo.taskRequires.values: result &= requirements +proc name*(pkgInfo: PackageInfo): string {.inline.} = + pkgInfo.basicInfo.name + +iterator lockedDepsFor*(pkgInfo: PackageInfo, options: Options): (string, LockFileDep) = + for task, deps in pkgInfo.lockedDeps: + if task in ["", options.task]: + for name, dep in deps: + yield (name, dep) + +proc hasLockedDeps*(pkgInfo: PackageInfo): bool = + ## Returns true if pkgInfo has any locked deps (including any tasks) + # Check if any tasks have locked deps + for deps in pkgInfo.lockedDeps.values: + if deps.len > 0: + return true + +proc hasPackage*(deps: AllLockFileDeps, pkgName: string): bool = + for deps in deps.values: + if pkgName in deps: + return true proc `==`*(pkg1: PackageInfo, pkg2: PackageInfo): bool = pkg1.myPath == pkg2.myPath @@ -537,6 +559,9 @@ proc hash*(x: PackageInfo): Hash = proc getNameAndVersion*(pkgInfo: PackageInfo): string = &"{pkgInfo.basicInfo.name}@{pkgInfo.basicInfo.version}" +proc isNim*(name: string): bool = + result = name == "nim" or name == "nimrod" or name == "compiler" + when isMainModule: import unittest diff --git a/src/nimblepkg/packageinfotypes.nim b/src/nimblepkg/packageinfotypes.nim index 508e7520..8f2cf531 100644 --- a/src/nimblepkg/packageinfotypes.nim +++ b/src/nimblepkg/packageinfotypes.nim @@ -21,6 +21,10 @@ type LockFileDeps* = OrderedTable[string, LockFileDep] + AllLockFileDeps* = Table[string, LockFileDeps] + ## Base deps is stored with empty string key "" + ## Other tasks have task name as key + PackageMetaData* = object url*: string downloadMethod*: DownloadMethod @@ -62,7 +66,7 @@ type backend*: string foreignDeps*: seq[string] basicInfo*: PackageBasicInfo - lockedDeps*: LockFileDeps + lockedDeps*: AllLockFileDeps metaData*: PackageMetaData isLink*: bool @@ -81,3 +85,5 @@ type alias*: string ## A name of another package, that this package aliases. PackageDependenciesInfo* = tuple[deps: HashSet[PackageInfo], pkg: PackageInfo] + +const noTask* = "" # Means that noTask is being ran. Use this as key for base dependencies diff --git a/src/nimblepkg/packageparser.nim b/src/nimblepkg/packageparser.nim index 039eb7e0..9ee75c2c 100644 --- a/src/nimblepkg/packageparser.nim +++ b/src/nimblepkg/packageparser.nim @@ -184,7 +184,9 @@ proc validatePackageInfo(pkgInfo: PackageInfo, options: Options) = raise validationError("'" & pkgInfo.backend & "' is an invalid backend.", false) - validatePackageStructure(pkginfo, options) + # nim is used for building the project, thus no need to validate its structure. + if not pkgInfo.basicInfo.name.isNim: + validatePackageStructure(pkginfo, options) proc nimScriptHint*(pkgInfo: PackageInfo) = if not pkgInfo.isNimScript: @@ -323,7 +325,7 @@ proc inferInstallRules(pkgInfo: var PackageInfo, options: Options) = # installed.) let installInstructions = pkgInfo.installDirs.len + pkgInfo.installExt.len + pkgInfo.installFiles.len - if installInstructions == 0 and pkgInfo.bin.len > 0: + if installInstructions == 0 and pkgInfo.bin.len > 0 and pkgInfo.basicInfo.name != "nim": pkgInfo.skipExt.add("nim") # When a package doesn't specify a `srcDir` it's fair to assume that diff --git a/src/nimblepkg/tools.nim b/src/nimblepkg/tools.nim index 29b7027b..e3688102 100644 --- a/src/nimblepkg/tools.nim +++ b/src/nimblepkg/tools.nim @@ -62,11 +62,6 @@ proc tryDoCmdEx*(cmd: string): string {.discardable.} = raise nimbleError(tryDoCmdExErrorMessage(cmd, output, exitCode)) return output -proc getNimBin*: string = - result = "nim" - if findExe("nim") != "": result = findExe("nim") - elif findExe("nimrod") != "": result = findExe("nimrod") - proc getNimrodVersion*(options: Options): Version = let vOutput = doCmdEx(getNimBin(options).quoteShell & " -v").output var matches: array[0..MaxSubpatterns, string] diff --git a/src/nimblepkg/topologicalsort.nim b/src/nimblepkg/topologicalsort.nim index 38dead23..8e0c1fef 100644 --- a/src/nimblepkg/topologicalsort.nim +++ b/src/nimblepkg/topologicalsort.nim @@ -11,7 +11,7 @@ proc getDependencies(packages: seq[PackageInfo], package: PackageInfo, ## package. It is needed because some of the names of the packages in the ## `requires` clause of a package could be URLs. for dep in package.requires: - if dep.name == "nim": + if dep.name.isNim: continue var depPkgInfo = initPackageInfo() var found = findPkg(packages, dep, depPkgInfo) diff --git a/tests/nimdep/nimdep.nimble b/tests/nimdep/nimdep.nimble new file mode 100644 index 00000000..7da37d48 --- /dev/null +++ b/tests/nimdep/nimdep.nimble @@ -0,0 +1,15 @@ +# Package + +version = "0.1.0" +author = "Ivan Yonchovski" +description = "A new awesome nimble package" +license = "MIT" +srcDir = "src" +bin = @["demo"] + +# Dependencies + +requires "nim" + +task version, "Test nim version": + exec "nim --version" diff --git a/tests/nimdep/src/demo.nim b/tests/nimdep/src/demo.nim new file mode 100644 index 00000000..e69de29b diff --git a/tests/taskdeps/dependencies/dependencies.nimble b/tests/taskdeps/dependencies/dependencies.nimble new file mode 100644 index 00000000..06f1ec62 --- /dev/null +++ b/tests/taskdeps/dependencies/dependencies.nimble @@ -0,0 +1,15 @@ +# Package + +version = "0.1.0" +author = "John Doe" +description = "A new awesome nimble package" +license = "MIT" +srcDir = "." +bin = @["foo"] + + +# Dependencies + +requires "nim >= 0.19.0" + +taskRequires "test", "unittest2 == 0.0.4" diff --git a/tests/taskdeps/main/foo.nim b/tests/taskdeps/main/foo.nim new file mode 100644 index 00000000..4b703fb7 --- /dev/null +++ b/tests/taskdeps/main/foo.nim @@ -0,0 +1 @@ +import json_serialization diff --git a/tests/taskdeps/main/main.nimble b/tests/taskdeps/main/main.nimble index a1b3b6bd..cfc876f2 100644 --- a/tests/taskdeps/main/main.nimble +++ b/tests/taskdeps/main/main.nimble @@ -4,7 +4,7 @@ version = "0.1.0" author = "John Doe" description = "A new awesome nimble package" license = "MIT" -srcDir = "." +srcDir = "src" bin = @[] diff --git a/tests/taskdeps/main/src/main.nim b/tests/taskdeps/main/src/main.nim new file mode 100644 index 00000000..e69de29b diff --git a/tests/tasks/getpaths/getpaths.nimble b/tests/tasks/getpaths/getpaths.nimble new file mode 100644 index 00000000..7b56d241 --- /dev/null +++ b/tests/tasks/getpaths/getpaths.nimble @@ -0,0 +1,17 @@ +# Package + +version = "0.1.0" +author = "Ivan Yonchovski" +description = "A new awesome nimble package" +license = "MIT" +srcDir = "src" +bin = @["run"] + + +# Dependencies + +requires "benchy", "unittest2" + +task echoPaths, "": + echo getPaths() + echo getPathsClause() diff --git a/tests/testscommon.nim b/tests/testscommon.nim index 6ba398ec..4729588e 100644 --- a/tests/testscommon.nim +++ b/tests/testscommon.nim @@ -135,7 +135,8 @@ proc uninstallDeps*() = for line in output.splitLines: let package = line.split(" ")[0] if package != "": - verify execNimbleYes("uninstall", "-i", package) + discard execNimbleYes("uninstall", "-i", package) + template testRefresh*(body: untyped) = # Backup current config diff --git a/tests/tgetpaths.nim b/tests/tgetpaths.nim new file mode 100644 index 00000000..9c3a66d6 --- /dev/null +++ b/tests/tgetpaths.nim @@ -0,0 +1,17 @@ +# Copyright (C) Dominik Picheta. All rights reserved. +# BSD License. Look at license.txt for more info. + +{.used.} + +import unittest, strutils, os +import testscommon +from nimblepkg/common import cd + +suite "nimble getPaths/getPathsClause": + test "check getPaths result": + cd "tasks/getpaths": + let (output, exitCode) = execNimble("echoPaths") + check output.contains("--path:") + check output.contains("benchy") + check output.contains("unittest2") + check exitCode == QuitSuccess diff --git a/tests/tinitcommand.nim b/tests/tinitcommand.nim index 8972427f..d6d4240f 100644 --- a/tests/tinitcommand.nim +++ b/tests/tinitcommand.nim @@ -9,6 +9,7 @@ suite "init": ## https://github.com/nim-lang/nimble/pull/983 test "init within directory that is invalid package name will not create new directory": let tempdir = getTempDir() / "a-b" + if dirExists tempdir: removeDir(tempDir) createDir(tempdir) cd(tempdir): let args = ["init"] diff --git a/tests/tlockfile.nim b/tests/tlockfile.nim index 32f70dd9..dcd8ff7d 100644 --- a/tests/tlockfile.nim +++ b/tests/tlockfile.nim @@ -172,6 +172,7 @@ requires "nim >= 1.5.1" let json = lockFileName.readFile.parseJson for (depName, depPath) in deps: let expectedVcsRevision = depPath.getVcsRevision + check depName in json{$lfjkPackages} let lockedVcsRevision = json{$lfjkPackages}{depName}{$lfjkPkgVcsRevision}.str.initSha1Hash check lockedVcsRevision == expectedVcsRevision @@ -590,3 +591,35 @@ requires "nim >= 1.5.1" writeDevelopFile(developFileName, @[], @[dep1PkgRepoPath, mainPkgOriginRepoPath]) let (_, exitCode) = execNimbleYes("--debug", "--verbose", "sync") check exitCode == QuitSuccess + proc getRevision(dep: string, lockFileName = defaultLockFileName): string = + result = lockFileName.readFile.parseJson{$lfjkPackages}{dep}{$lfjkPkgVcsRevision}.str + + test "can generate lock file for nim as dep": + cleanUp() + cd "nimdep": + removeFile "nimble.develop" + removeFile "nimble.lock" + removeDir "Nim" + + check execNimbleYes("develop", "nim").exitCode == QuitSuccess + cd "Nim": + let (_, exitCode) = execNimbleYes("-y", "install") + check exitCode == QuitSuccess + + # check if the compiler version will be used when doing build + testLockFile(@[("nim", "Nim")], isNew = true) + removeFile "nimble.develop" + removeDir "Nim" + + let (output, exitCodeInstall) = execNimbleYes("-y", "build") + check exitCodeInstall == QuitSuccess + let usingNim = when defined(Windows): "nim.exe for compilation" else: "bin/nim for compilation" + check output.contains(usingNim) + + # check the nim version + let (outputVersion, _) = execNimble("version") + check outputVersion.contains(getRevision("nim")) + + let (outputGlobalNim, exitCodeGlobalNim) = execNimbleYes("-y", "--use-system-nim", "build") + check exitCodeGlobalNim == QuitSuccess + check not outputGlobalNim.contains(usingNim) diff --git a/tests/ttaskdeps.nim b/tests/ttaskdeps.nim index 616243e1..6393a86b 100644 --- a/tests/ttaskdeps.nim +++ b/tests/ttaskdeps.nim @@ -10,58 +10,65 @@ from nimblepkg/common import cd template makeLockFile() = ## Makes lock file, cleans up after itself - verify execNimble("lock") + verify execNimbleYes("lock") defer: removeFile("nimble.lock") template inDir(body: untyped) = ## Runs code inside taskdeps folder cd "taskdeps/main/": + removeFile("nimble.lock") body suite "Task level dependencies": + uninstallDeps() verify execNimbleYes("update") - teardown: uninstallDeps() test "Can specify custom requirement for a task": inDir: - verify execNimble("tasks") + verify execNimbleYes("tasks") test "Dependency is used when running task": inDir: - let (output, exitCode) = execNimble("benchmark") + let (output, exitCode) = execNimbleYes("benchmark") check exitCode == QuitSuccess - check output.contains("dependencies for benchy@0.0.1") - check not output.contains("dependencies for unittest2@0.0.4") + check output.contains("benchy@0.0.1") + # Check other tasks aren't used + check not output.contains("unittest2@0.0.4") test "Dependency is not used when not running task": inDir: - let (output, exitCode) = execNimble("install") + let (output, exitCode) = execNimbleYes("install") check exitCode == QuitSuccess - check not output.contains("dependencies for unittest2@0.0.4") - check not output.contains("dependencies for benchy@0.0.1") + check not output.contains("unittest2@0.0.4") + check not output.contains("benchy@0.0.1") test "Dependency can be defined for test task": inDir: - let (output, exitCode) = execNimble("test") + let (output, exitCode) = execNimbleYes("test") check exitCode == QuitSuccess - check output.contains("dependencies for unittest2@0.0.4") + check output.contains("unittest2@0.0.4") test "Lock file has dependencies added to it": inDir: makeLockFile() # Check task level dependencies are in the lock file - let json = parseFile("nimble.lock") - check "unittest2" in json["packages"] - let pkgInfo = json["packages"]["unittest2"] - check pkgInfo["version"].getStr() == "0.0.4" + let + json = parseFile("nimble.lock") + tasks = json["tasks"] + packages = json["packages"] + check: + "test" in tasks + "benchmark" in tasks + "unittest2" notin packages + check tasks["test"]["unittest2"]["version"].getStr() == "0.0.4" test "Task dependencies from lock file are used": inDir: makeLockFile() uninstallDeps() - let (output, exitCode) = execNimble("test") + let (output, exitCode) = execNimbleYes("test") check exitCode == QuitSuccess check not output.contains("benchy installed successfully") check output.contains("unittest2 installed successfully") @@ -74,7 +81,7 @@ suite "Task level dependencies": # tries to install them later uninstallDeps() - let (output, exitCode) = execNimble("install") + let (output, exitCode) = execNimbleYes("install") check exitCode == QuitSuccess check not output.contains("benchy installed successfully") check not output.contains("unittest2 installed successfully") @@ -84,7 +91,7 @@ suite "Task level dependencies": # Uninstall the dependencies fist to make sure deps command # still installs everything correctly uninstallDeps() - let (output, exitCode) = execNimble("--format:json", "--silent", "deps") + let (output, exitCode) = execNimbleYes("--format:json", "--silent", "deps") check exitCode == QuitSuccess let json = parseJson(output) @@ -100,39 +107,57 @@ suite "Task level dependencies": removeDir("nim-unittest2") removeFile("nimble.develop") - verify execNimble("develop", "unittest2") + verify execNimbleYes("develop", "unittest2") # Add in a file to the develop file # We will then try and import this createDir "nim-unittest2/unittest2" "nim-unittest2/unittest2/customFile.nim".writeFile("") - let (output, exitCode) = execNimble("-d:useDevelop", "test") + let (output, exitCode) = execNimbleYes("-d:useDevelop", "test") check exitCode == QuitSuccess check "Using custom file" in output test "Dependencies aren't verified twice": inDir: - let (output, _) = execNimble("test") + let (output, _) = execNimbleYes("test") check output.count("dependencies for unittest2@0.0.4") == 1 test "Requirements for tasks in dependencies aren't used": cd "taskdeps/subdep/": + removeFile("nimble.lock") let (output, _) = execNimbleYes("install") check "threading" notin output inDir: - let (output, exitCode) = execNimble("test") + let (output, exitCode) = execNimbleYes("test") + check exitCode == QuitSuccess + check "threading" notin output + + test "Requirements for tasks in dependencies aren't used (When using lock file)": + cd "taskdeps/subdep/": + makeLockFile() + let (output, _) = execNimbleYes("install") + check "threading" notin output + + inDir: + let (output, exitCode) = execNimbleYes("test") check exitCode == QuitSuccess check "threading" notin output test "Error thrown when setting requirement for task that doesn't exist": cd "taskdeps/error/": - let (output, exitCode) = execNimble("check") + let (output, exitCode) = execNimbleYes("check") check exitCode == QuitFailure check "Task benchmark doesn't exist for requirement benchy == 0.0.1" in output test "Dump contains information": inDir: - let (output, exitCode) = execNimble("dump") + let (output, exitCode) = execNimbleYes("dump") check exitCode == QuitSuccess check output.processOutput.inLines("benchmarkRequires: \"benchy 0.0.1\"") check output.processOutput.inLines("testRequires: \"unittest2 0.0.4\"") + + test "Lock files don't break": + # Tests for regression caused by tasks deps. + # nimlangserver is good candidate, has locks and quite a few dependencies + let (_, exitCode) = execNimbleYes("install", "nimlangserver@#19715af") + check exitCode == QuitSuccess