Make special version a set of versions

Special versions are made a set of versions. Those are aliases with
which a single package can be referred. For example, a package can be
simultaneously versions:

 * 0.1.0 - the normal version from the Nimble file.

 * #head - the latest commit in the main branch

 * #master - the main branch name

 * 3c91b869 - part of the sha1 hash of the latest commit in the main
              branch

When the same package is downloaded a second time (determined by the
checksum) instead of proposing to replace it just print a warning that
the package is already installed and merge the special version of the
new package with a special version of the already installed one.

Additionally this commit:

 - Removes some legacy code for supporting the old package format in the
   reverse dependencies.

 - The names of the packages in the reverse dependencies are written
   without converting to lower case.

 - The tests are fixed according to the new behavior.

Related to nim-lang#127

src/nimble.nim

@@ -76,8 +76,8 @@ proc processFreeDependencies(pkgInfo: PackageInfo, options: Options):
   var pkgList {.global.}: seq[PackageInfo] = @[]
   once: pkgList = initPkgList(pkgInfo, options)
 
-  display("Verifying",
-          "dependencies for $1@$2" % [pkgInfo.basicInfo.name, $pkgInfo.metaData.specialVersion],
+  display("Verifying", "dependencies for $1@$2" %
+          [pkgInfo.basicInfo.name, $pkgInfo.basicInfo.version],
           priority = HighPriority)
 
   var reverseDependencies: seq[PackageBasicInfo] = @[]
@@ -106,7 +106,15 @@ proc processFreeDependencies(pkgInfo: PackageInfo, options: Options):
         let (packages, installedPkg) = install(toInstall, options,
           doPrompt = false, first = false, fromLockFile = false)
 
-        result.incl packages
+        for pkg in packages:
+          if result.contains pkg:
+            # If the result already contains the newly tried to install package
+            # we had to merge its special versions set into the set of the old
+            # one.
+            result[pkg].metaData.specialVersions.incl(
+              pkg.metaData.specialVersions)
+          else:
+            result.incl pkg
 
         pkg = installedPkg # For addRevDep
         fillMetaData(pkg, pkg.getRealDir(), false)
@@ -119,7 +127,7 @@ proc processFreeDependencies(pkgInfo: PackageInfo, options: Options):
         # Process the dependencies of this dependency.
         result.incl processFreeDependencies(pkg.toFullInfo(options), options)
       if not pkg.isLink:
-        reverseDependencies.add((pkg.basicInfo.name, pkg.metaData.specialVersion, pkg.basicInfo.checksum))
+        reverseDependencies.add(pkg.basicInfo)
 
   # Check if two packages of the same name (but different version) are listed
   # in the path.
@@ -260,27 +268,24 @@ proc removePackage(pkgInfo: PackageInfo, options: Options) =
   reinstallSymlinksForOlderVersion(pkgDestDir, options)
   options.nimbleData.removeRevDep(pkgInfo)
 
-proc packageExists(pkgInfo: PackageInfo, options: Options): bool =
-  let pkgDestDir = pkgInfo.getPkgDest(options)
-  return fileExists(pkgDestDir / packageMetaDataFileName)
-
-proc promptOverwriteExistingPackage(pkgInfo: PackageInfo,
-                                    options: Options): bool =
-  let message = "$1@$2 already exists. Overwrite?" %
-                [pkgInfo.basicInfo.name, $pkgInfo.metaData.specialVersion]
-  return options.prompt(message)
-
-proc removeOldPackage(pkgInfo: PackageInfo, options: Options) =
+proc packageExists(pkgInfo: PackageInfo, options: Options):
+    Option[PackageInfo] =
+  ## Checks whether a package `pkgInfo` already exists in the Nimble cache. If a
+  ## package already exists returns the `PackageInfo` of the package in the
+  ## cache otherwise returns `none`. Raises a `NimbleError` in the case the
+  ## package exists in the cache but it is not valid.
   let pkgDestDir = pkgInfo.getPkgDest(options)
-  let oldPkgInfo = getPkgInfo(pkgDestDir, options)
-  removePackage(oldPkgInfo, options)
-
-proc promptRemovePackageIfExists(pkgInfo: PackageInfo, options: Options): bool =
-  if packageExists(pkgInfo, options):
-    if not promptOverwriteExistingPackage(pkgInfo, options):
-      return false
-    removeOldPackage(pkgInfo, options)
-  return true
+  if not fileExists(pkgDestDir / packageMetaDataFileName):
+    return none[PackageInfo]()
+  else:
+    var oldPkgInfo = initPackageInfo()
+    try:
+      oldPkgInfo = pkgDestDir.getPkgInfo(options)
+    except CatchableError as error:
+      raise nimbleError(&"The package inside \"{pkgDestDir}\" is invalid.",
+                        details = error)
+    fillMetaData(oldPkgInfo, pkgDestDir, true)
+    return some(oldPkgInfo)
 
 proc processLockedDependencies(pkgInfo: PackageInfo, options: Options):
   HashSet[PackageInfo]
@@ -329,9 +334,10 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
   var depsOptions = options
   depsOptions.depsOnly = false
 
-  # Overwrite the version if the requested version is "#head" or similar.
   if requestedVer.kind == verSpecial:
-    pkgInfo.metaData.specialVersion = requestedVer.spe
+    # Add a version alias to special versions set if requested version is a
+    # special one.
+    pkgInfo.metaData.specialVersions.incl requestedVer.spe
 
   # Dependencies need to be processed before the creation of the pkg dir.
   if first and pkgInfo.lockedDeps.len > 0:
@@ -343,10 +349,24 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
     result.pkg = pkgInfo
     return result
 
-  display("Installing", "$1@$2" % [pkginfo.basicInfo.name, $pkginfo.metaData.specialVersion],
-          priority = HighPriority)
+  display("Installing", "$1@$2" %
+    [pkginfo.basicInfo.name, $pkginfo.basicInfo.version],
+    priority = HighPriority)
 
-  let isPackageAlreadyInCache = pkgInfo.packageExists(options)
+  let oldPkg = pkgInfo.packageExists(options)
+  if oldPkg.isSome:
+    # In the case we already have the same package in the cache then only merge
+    # the new package special versions to the old one.
+    displayWarning(pkgAlreadyExistsInTheCacheMsg(pkgInfo))
+    var oldPkg = oldPkg.get
+    oldPkg.metaData.specialVersions.incl pkgInfo.metaData.specialVersions
+    saveMetaData(oldPkg.metaData, oldPkg.getNimbleFileDir, changeRoots = false)
+    if result.deps.contains oldPkg:
+      result.deps[oldPkg].metaData.specialVersions.incl(
+        oldPkg.metaData.specialVersions)
+    result.deps.incl oldPkg
+    result.pkg = oldPkg
+    return
 
   # Build before removing an existing package (if one exists). This way
   # if the build fails then the old package will still be installed.
@@ -361,8 +381,7 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
     try:
       buildFromDir(pkgInfo, paths, "-d:release" & flags, options)
     except CatchableError:
-      if not isPackageAlreadyInCache:
-        removeRevDep(options.nimbleData, pkgInfo)
+      removeRevDep(options.nimbleData, pkgInfo)
       raise
 
   let pkgDestDir = pkgInfo.getPkgDest(options)
@@ -374,9 +393,6 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options,
   # Don't copy artifacts if project local deps mode and "installing" the top
   # level package.
   if not (options.localdeps and options.isInstallingTopLevel(dir)):
-    if not promptRemovePackageIfExists(pkgInfo, options):
-      return
-
     createDir(pkgDestDir)
     # Copy this package's files based on the preferences specified in PkgInfo.
     var filesInstalled: HashSet[string]
@@ -797,18 +813,22 @@ proc list(options: Options) =
     echo(" ")
 
 proc listInstalled(options: Options) =
-  var h: OrderedTable[string, seq[Version]]
+  type
+    VersionChecksumTuple = tuple[version: Version, checksum: Sha1Hash]
+  var h: OrderedTable[string, seq[VersionChecksumTuple]]
   let pkgs = getInstalledPkgsMin(options.getPkgsDir(), options)
   for pkg in pkgs:
     let
       pName = pkg.basicInfo.name
-      pVer = pkg.metaData.specialVersion
+      pVersion = pkg.basicInfo.version
+      pChecksum = pkg.basicInfo.checksum
     if not h.hasKey(pName): h[pName] = @[]
     var s = h[pName]
-    add(s, pVer)
+    add(s, (pVersion, pChecksum))
     h[pName] = s
 
-  h.sort(proc (a, b: (string, seq[Version])): int = cmpIgnoreCase(a[0], b[0]))
+  h.sort(proc (a, b: (string, seq[VersionChecksumTuple])): int =
+    cmpIgnoreCase(a[0], b[0]))
   for k in keys(h):
     echo k & "  [" & h[k].join(", ") & "]"
 
@@ -837,7 +857,7 @@ proc listPaths(options: Options) =
     # There may be several, list all available ones and sort by version.
     for pkg in pkgs:
       if name == pkg.basicInfo.name:
-        installed.add((pkg.metaData.specialVersion, pkg.getRealDir))
+        installed.add((pkg.basicInfo.version, pkg.getRealDir))
 
     if installed.len > 0:
       sort(installed, cmp[VersionAndPath], Descending)
@@ -1127,7 +1147,7 @@ proc uninstall(options: var Options) =
         if len(revDeps - pkgsToDelete) > 0:
           let pkgs = revDeps.collectNames(true)
           displayWarning(
-            cannotUninstallPkgMsg(pkgTup.name, pkg.metaData.specialVersion, pkgs))
+            cannotUninstallPkgMsg(pkgTup.name, pkg.basicInfo.version, pkgs))
         else:
           pkgsToDelete.incl pkg.toRevDep
 

src/nimblepkg/displaymessages.nim

@@ -6,7 +6,7 @@
 ## the message to be repeated both in Nimble and the testing code.
 
 import strformat, strutils
-import version
+import version, packageinfotypes, sha1hashes
 
 const
   validationFailedMsg* = "Validation failed."
@@ -143,3 +143,13 @@ proc invalidDevelopDependenciesVersionsMsg*(errors: seq[string]): string =
   for error in errors:
     result &= "\n"
     result &= error
+
+proc pkgAlreadyExistsInTheCacheMsg*(name, version, checksum: string): string =
+  &"A package \"{name}@{version}\" with checksum \"{checksum}\" already " &
+   "exists the the cache."
+
+proc pkgAlreadyExistsInTheCacheMsg*(pkgInfo: PackageInfo): string =
+  pkgAlreadyExistsInTheCacheMsg(
+     pkgInfo.basicInfo.name,
+    $pkgInfo.basicInfo.version,
+    $pkgInfo.basicInfo.checksum)

src/nimblepkg/packageinfo.nim

@@ -279,7 +279,7 @@ proc setNameVersionChecksum*(pkgInfo: var PackageInfo, pkgDir: string) =
   if pkgInfo.basicInfo.version == notSetVersion:
     # if there is no previously set version from the `.nimble` file
     pkgInfo.basicInfo.version = version
-  pkgInfo.metaData.specialVersion = version
+  pkgInfo.metaData.specialVersions.incl version
   pkgInfo.basicInfo.checksum = checksum
 
 proc getInstalledPackageMin*(pkgDir, nimbleFilePath: string): PackageInfo =
@@ -304,11 +304,10 @@ proc getInstalledPkgsMin*(libsDir: string, options: Options): seq[PackageInfo] =
         result.add pkg
 
 proc withinRange*(pkgInfo: PackageInfo, verRange: VersionRange): bool =
-  ## Determines whether the specified package's version is within the
-  ## specified range. The check works with ordinary versions as well as
-  ## special ones.
-  return withinRange(pkgInfo.basicInfo.version, verRange) or
-         withinRange(pkgInfo.metaData.specialVersion, verRange)
+  ## Determines whether the specified package's version is within the specified
+  ## range. As the ordinary version is always added to the special versions set
+  ## checking only the special versions is enough.
+  return withinRange(pkgInfo.metaData.specialVersions, verRange)
 
 proc resolveAlias*(dep: PkgTuple, options: Options): PkgTuple =
   ## Looks up the specified ``dep.name`` in the packages.json files to resolve
@@ -496,7 +495,7 @@ proc iterInstallFiles*(realDir: string, pkgInfo: PackageInfo,
         action(file)
 
 proc getCacheDir*(pkgInfo: PackageBasicInfo): string =
-  &"{pkgInfo.name}-{pkgInfo.version}-{pkgInfo.checksum}"
+  &"{pkgInfo.name}-{pkgInfo.version}-{$pkgInfo.checksum}"
 
 proc getPkgDest*(pkgInfo: PackageBasicInfo, options: Options): string =
   options.getPkgsDir() / pkgInfo.getCacheDir()

src/nimblepkg/packageinfotypes.nim

@@ -27,7 +27,10 @@ type
     vcsRevision*: Sha1Hash
     files*: seq[string]
     binaries*: seq[string]
-    specialVersion*: Version
+    specialVersions*: HashSet[Version]
+      # Special versions are aliases with which a single package can be
+      # referred. For example a package can be versions `0.1.0`, `#head` and
+      # `#master` at the same time.
 
   PackageBasicInfo* = tuple
     name: string

src/nimblepkg/packagemetadatafile.nim

@@ -1,7 +1,7 @@
 # Copyright (C) Dominik Picheta. All rights reserved.
 # BSD License. Look at license.txt for more info.
 
-import json, os, strformat
+import json, os, strformat, sets, sequtils
 import common, version, packageinfotypes, cli, tools, sha1hashes
 
 type
@@ -17,20 +17,40 @@ const
 
 proc initPackageMetaData*(): PackageMetaData =
   result = PackageMetaData(
-    specialVersion: notSetVersion,
     vcsRevision: notSetSha1Hash)
 
 proc metaDataError(msg: string): ref MetaDataError =
   newNimbleError[MetaDataError](msg)
 
-proc saveMetaData*(metaData: PackageMetaData, dirName: string) =
+proc `%`(specialVersions: HashSet[Version]): JsonNode =
+  %specialVersions.toSeq
+
+proc initFromJson(specialVersions: var HashSet[Version], jsonNode: JsonNode,
+                  jsonPath: var string) =
+  case jsonNode.kind
+  of JArray:
+    let originalJsonPathLen = jsonPath.len
+    for i in 0 ..< jsonNode.len:
+      jsonPath.add '['
+      jsonPath.addInt i
+      jsonPath.add ']'
+      var version = newVersion("")
+      initFromJson(version, jsonNode[i], jsonPath)
+      specialVersions.incl version
+      jsonPath.setLen originalJsonPathLen
+  else:
+    assert false, "The `jsonNode` must be of kind JArray."
+
+proc saveMetaData*(metaData: PackageMetaData, dirName: string,
+                   changeRoots = true) =
   ## Saves some important data to file in the package installation directory.
   var metaDataWithChangedPaths = metaData
-  for i, file in metaData.files:
-    metaDataWithChangedPaths.files[i] = changeRoot(dirName, "", file)
+  if changeRoots:
+    for i, file in metaData.files:
+      metaDataWithChangedPaths.files[i] = changeRoot(dirName, "", file)
   let json = %{
     $pmdjkVersion: %packageMetaDataFileVersion,
-    $pmdjkMetaData: %metaDataWithChangedPaths }
+    $pmdjkMetaData: %metaDataWithChangedPaths}
   writeFile(dirName / packageMetaDataFileName, json.pretty)
 
 proc loadMetaData*(dirName: string, raiseIfNotFound: bool): PackageMetaData =
@@ -39,7 +59,9 @@ proc loadMetaData*(dirName: string, raiseIfNotFound: bool): PackageMetaData =
   let fileName = dirName / packageMetaDataFileName
   if fileExists(fileName):
     {.warning[ProveInit]: off.}
+    {.warning[UnsafeSetLen]: off.}
     result = parseFile(fileName)[$pmdjkMetaData].to(PackageMetaData)
+    {.warning[UnsafeSetLen]: on.}
     {.warning[ProveInit]: on.}
   elif raiseIfNotFound:
     raise metaDataError(&"No {packageMetaDataFileName} file found in {dirName}")

src/nimblepkg/packageparser.nim

@@ -383,7 +383,7 @@ proc readPackageInfo(nf: NimbleFile, options: Options, onlyMinimalInfo=false):
     # some of the package meta data from its directory.
     result.basicInfo.checksum = calculateDirSha1Checksum(fileDir)
     # By default specialVersion is the same as version.
-    result.metaData.specialVersion = result.basicInfo.version
+    result.metaData.specialVersions.incl result.basicInfo.version
     # If the `fileDir` is a VCS repository we can get some of the package meta
     # data from it.
     result.metaData.vcsRevision = getVcsRevision(fileDir)
@@ -497,7 +497,7 @@ proc toFullInfo*(pkg: PackageInfo, options: Options): PackageInfo =
     # The `isLink` data from the meta data file is with priority because of the
     # old format develop packages.
     result.isLink = pkg.isLink
-    result.metaData.specialVersion = pkg.metaData.specialVersion
+    result.metaData.specialVersions.incl pkg.metaData.specialVersions
 
     assert not (pkg.isInstalled and pkg.isLink),
            "A package must not be simultaneously installed and linked."