yusdacra improvements

crate2nix/src/config.rs

@@ -104,8 +104,10 @@ pub enum Source {
         url: url::Url,
         /// The revision hash.
         rev: String,
+        /// The reference (branch or tag).
+        r#ref: Option<String>,
         /// The sha256 of the fetched result.
-        sha256: String,
+        sha256: Option<String>,
     },
     /// Get the source from a nix expression.
     Nix {
@@ -181,7 +183,7 @@ impl Display for Source {
                 version,
                 sha256,
             } => write!(f, "{} {} from crates.io: {}", name, version, sha256),
-            Source::Git { url, rev, sha256 } => write!(f, "{}#{} via git: {}", url, rev, sha256),
+            Source::Git { url, rev, r#ref, sha256 } => write!(f, "{}#{} on {:?} via git: {:?}", url, rev, r#ref, sha256),
             Source::Nix { file, attr: None } => write!(f, "{}", file),
             Source::Nix {
                 file,

crate2nix/src/main.rs

@@ -267,6 +267,9 @@ pub enum SourceAddingCommands {
 
         #[structopt(long = "rev", parse(from_str), help = "The git revision hash.")]
         rev: String,
+
+        #[structopt(long = "ref", parse(from_str), help = "The git reference (branch or tag).")]
+        r#ref: Option<String>,
     },
 
     #[structopt(
@@ -319,8 +322,8 @@ impl SourceAddingCommands {
                 let source = crate2nix::sources::crates_io_source(crate_name, crate_version)?;
                 (name, source)
             }
-            SourceAddingCommands::Git { name, url, rev } => {
-                let source = crate2nix::sources::git_io_source(url, rev)?;
+            SourceAddingCommands::Git { name, url, rev, r#ref } => {
+                let source = crate2nix::sources::git_io_source(url, rev, r#ref)?;
                 (name, source)
             }
             SourceAddingCommands::Nix {

crate2nix/src/resolve.rs

@@ -366,11 +366,11 @@ pub enum ResolvedSource {
 impl From<crate::config::Source> for ResolvedSource {
     fn from(source: crate::config::Source) -> Self {
         match source {
-            crate::config::Source::Git { url, rev, sha256 } => ResolvedSource::Git(GitSource {
+            crate::config::Source::Git { url, rev, r#ref, sha256 } => ResolvedSource::Git(GitSource {
                 url,
                 rev,
-                r#ref: None,
-                sha256: Some(sha256),
+                r#ref,
+                sha256,
             }),
             crate::config::Source::CratesIo {
                 name,
@@ -459,8 +459,8 @@ impl ResolvedSource {
         let mut url = url::Url::parse(&source_string[GIT_SOURCE_PREFIX.len()..])?;
         let mut query_pairs = url.query_pairs();
 
-        let branch = query_pairs
-            .find(|(k, _)| k == "branch")
+        let r#ref = query_pairs
+            .find(|(k, _)| k == "branch" || k == "tag")
             .map(|(_, v)| v.to_string());
         let rev = if let Some((_, rev)) = query_pairs.find(|(k, _)| k == "rev") {
             rev.to_string()
@@ -479,7 +479,7 @@ impl ResolvedSource {
         Ok(ResolvedSource::Git(GitSource {
             url,
             rev,
-            r#ref: branch,
+            r#ref,
             sha256: None,
         }))
     }

crate2nix/src/sources.rs

@@ -34,19 +34,19 @@ pub fn crates_io_source(name: String, version: Version) -> Result<config::Source
 }
 
 /// Returns the completed Source::Git definition by prefetching the hash.
-pub fn git_io_source(url: Url, rev: String) -> Result<config::Source, Error> {
+pub fn git_io_source(url: Url, rev: String, r#ref: Option<String>) -> Result<config::Source, Error> {
     let prefetchable = GitSource {
         url: url.clone(),
         rev: rev.clone(),
-        r#ref: None,
+        r#ref: r#ref.clone(),
         sha256: None,
     };
 
     eprint!("Prefetching {}: ", prefetchable.to_string());
     let sha256 = prefetchable.prefetch()?;
     eprintln!("done.");
 
-    Ok(config::Source::Git { url, rev, sha256 })
+    Ok(config::Source::Git { url, rev, r#ref, sha256: Some(sha256) })
 }
 
 /// Operations on assmebling out-of-tree sources via nix.

crate2nix/templates/Cargo.nix.tera

@@ -130,13 +130,15 @@ rec {
           else {{crate.source.LocalDirectory.path | safe}};
         {%- elif crate.source.Git %}
         workspace_member = null;
-        src = pkgs.fetchgit {
+        src = builtins.fetchGit ({
           url = {{crate.source.Git.url}};
           rev = {{crate.source.Git.rev}};
-          {%- if crate.source.Git.sha256 %}
-          sha256 = {{ crate.source.Git.sha256 }};
-          {%- endif %}
-        };
+          submodules = true;
+        } // (if isNull {% if crate.source.Git.ref %} {{crate.source.Git.ref}} {% else %} null {% endif %} then {
+          allRefs = true;
+        } else {
+          ref = {% if crate.source.Git.ref %} {{crate.source.Git.ref}} {% else %} null {% endif %};
+        }));
         {%- else %}
         src = builtins.throw ''ERROR: Could not resolve source: {{crate.source | json_encode() | safe}}'';
         {%- endif -%}

crate2nix/templates/crate2nix-sources.nix.tera

@@ -40,11 +40,14 @@ rec {
             assert builtins.isString type;
 
             if type == "Git"
-            then pkgs.fetchgit {
+            then builtins.fetchGit ({
                 url = source.url;
                 rev = source.rev;
-                sha256 = source.sha256;
-            }
+                submodules = true;
+            } // (if isNull source.ref
+                then { allRefs = true; } 
+                else { inherit (source) ref; }
+            ))
             else if type == "CratesIo"
             then downloadFromCratesIo source
             else if type == "Nix"
@@ -108,4 +111,4 @@ rec {
                 tar -xzf ${archive} --strip-components=1 -C $out
             '';
     };
-}
+}

tests.nix

@@ -759,10 +759,10 @@ in
   #
   # It is to have them directly as attributes for testing.
 
-  registryGit = pkgs.fetchgit {
+  registryGit = builtins.fetchGit {
     url = "https://github.com/rust-lang/crates.io-index";
     rev = "18e3f063f594fc08a078f0de2bb3f94beed16ae2";
-    sha256 = "0rpv12ifgnni55phlkb5ppmala7y3zrsc9dl8l99pbsjpqx95vmj";
+    allRefs = true;
   };
 
   registry = pkgs.linkFarm "crates.io-index" [

tools.nix

@@ -162,20 +162,51 @@ rec {
         builtins.throw "unknown source type: ${source}";
 
     # Extracts URL and rev from a git source URL.
-    #
-    # Crude, should be more robust :(
     parseGitSource = source:
       assert builtins.isString source;
       let
-        withoutGitPlus = lib.removePrefix "git+" source;
-        splitHash = lib.splitString "#" withoutGitPlus;
-        splitQuestion = lib.concatMap (lib.splitString "?") splitHash;
+        extractRevision = source: lib.last (lib.splitString "#" source);
+        extractPart = part: source: if lib.hasInfix part source then lib.last (lib.splitString part (lib.head (lib.splitString "#" source))) else null;
+        extractRepoUrl = source:
+          let
+            splitted = lib.head (lib.splitString "?" source);
+            split = lib.substring 4 (lib.stringLength splitted) splitted;
+          in
+          lib.head (lib.splitString "#" split);
+
+        revision = extractRevision source;
+        rev = extractPart "?rev=" source;
+        tag = extractPart "?tag=" source;
+        branch = extractPart "?branch=" source;
+        url = extractRepoUrl source;
       in
       {
-        url = builtins.head splitQuestion;
-        rev = lib.last splitQuestion;
+        inherit revision url;
+      } // lib.optionalAttrs (! isNull rev) {
+        inherit rev;
+      } // lib.optionalAttrs (! isNull tag) {
+        inherit tag;
+      } // lib.optionalAttrs (! isNull branch) {
+        inherit branch;
       };
 
+    getSrcFromGitSource = source:
+      let
+        parsed = parseGitSource source;
+        ref = parsed.branch or parsed.tag or null;
+      in builtins.fetchGit ({
+        inherit (parsed) url;
+        rev = parsed.revision;
+        submodules = true;
+      } // (if isNull ref then {
+        allRefs = true;
+      } else {
+        ref =
+          if ref == (parsed.tag or null)
+          then "refs/tags/${ref}"
+          else ref;
+      }));
+
     vendorSupport = { crateDir ? ./., ... }:
       rec {
         toPackageId = { name, version, source, ... }:
@@ -231,12 +262,7 @@ rec {
 
         mkGitHash = { source, ... }@attrs:
           let
-            parsed = parseGitSource source;
-            src = builtins.fetchGit {
-              submodules = true;
-              inherit (parsed) url rev;
-              ref = attrs.branch or "master";
-            };
+            src = getSrcFromGitSource source;
             hash = pkgs.runCommand "hash-of-${attrs.name}" { nativeBuildInputs = [ pkgs.nix ]; } ''
               echo -n "$(nix-hash --type sha256 ${src})" > $out
             '';
@@ -299,19 +325,41 @@ rec {
                 assert builtins.isString source;
                 let
                   parsed = parseGitSource source;
+                  has = name: attrs: ! isNull (attrs.${name} or null);
+                  hasTag = has "tag";
+                  hasRev = has "rev";
+                  hasBranch = has "branch";
+                  putBranch = (hasBranch attrs) || (hasBranch parsed);
+                  putTag = (hasTag attrs) || (hasTag parsed);
+                  putRev = (hasRev attrs) || (hasRev parsed);
+                  isNewerCargo = builtins.compareVersions pkgs.cargo.version "1.53.0" > (-1);
+                  key = if putRev    then "?rev=${attrs.rev or parsed.rev}"       else
+                        if putTag    then "?tag=${attrs.tag or parsed.tag}"       else
+                        if putBranch then "?branch=${attrs.branch or parsed.branch}" else
+                        "";
                 in
                 ''
 
-              [source."${parsed.url}"]
+              [source."${parsed.url}${key}"]
               git = "${parsed.url}"
-              rev = "${parsed.rev}"
-              ${lib.optionalString (isNull (builtins.match ".*\\?rev=[0-9a-z]{40}.*" source)) ''branch = "${attrs.branch or "master"}"''}
+              ${
+                if isNewerCargo
+                then lib.optionalString putRev ''rev = "${attrs.rev or parsed.rev or parsed.revision}"''
+                else ''rev = "${attrs.rev or parsed.rev or parsed.revision}"''
+              }
+              ${
+                if isNewerCargo
+                then lib.optionalString ((! putRev) && putBranch) ''branch = "${attrs.branch or parsed.branch}"''
+                else lib.optionalString ((! putRev) && (! putTag)) ''branch = "${attrs.branch or parsed.branch or "master"}"''
+              }
+              ${lib.optionalString ((! putRev) && putTag) ''tag = "${attrs.tag or parsed.tag}"''}
               replace-with = "vendored-sources"
               '';
             gitSources = packagesByType."git" or [ ];
             gitSourcesUnique = lib.unique gitSources;
             gitSourceConfigs = builtins.map gitSourceConfig gitSourcesUnique;
-            gitSourceConfigsString = lib.concatStrings gitSourceConfigs;
+            gitSourceConfigsUnique = lib.unique gitSourceConfigs;
+            gitSourceConfigsString = lib.concatStrings gitSourceConfigsUnique;
           in
           pkgs.writeText
             "vendor-config"
@@ -350,21 +398,40 @@ rec {
             assert (sourceType package) == "git";
             let
               packageId = toPackageId package;
-              sha256 =
-                hashes.${packageId}
-                  or extraHashes.${packageId}
-                  or (builtins.throw "Checksum for ${packageId} not found in crate-hashes.json");
-              parsed = parseGitSource source;
-              src = pkgs.fetchgit {
-                name = "${name}-${version}";
-                inherit sha256;
-                inherit (parsed) url rev;
-              };
+              src = getSrcFromGitSource source;
+              srcName = "${name}-${version}";
+
+              rootCargo = builtins.fromTOML (builtins.readFile "${src}/Cargo.toml");
+              isWorkspace = rootCargo ? "workspace";
+              isPackage = rootCargo ? "package";
+              containedCrates = rootCargo.workspace.members ++ (if isPackage then [ "." ] else [ ]);
+
+              getCrateNameFromPath = path:
+                let
+                  cargoTomlCrate = builtins.fromTOML (builtins.readFile "${src}/${path}/Cargo.toml");
+                in
+                cargoTomlCrate.package.name;
+
+              filteredPaths =
+                builtins.filter
+                  (to_filter: getCrateNameFromPath to_filter == name)
+                  containedCrates;
+
+              pathToExtract =
+                if isWorkspace then
+                  # Workaround for sources that have isWorkspace as true, but don't
+                  # declare all their members in `workspace.members`
+                  if builtins.length filteredPaths > 0
+                  then builtins.head filteredPaths
+                  # This does not cover all possible cases, only is a last ditch effort
+                  else name
+                else
+                  ".";
             in
-            pkgs.runCommand (lib.removeSuffix ".tar.gz" src.name) { }
+            pkgs.runCommand (lib.removeSuffix ".tar.gz" srcName) { }
               ''
                 mkdir -p $out
-                cp -apR ${src}/* $out
+                cp -apR ${src}/${pathToExtract}/* $out
                 echo '{"package":null,"files":{}}' > $out/.cargo-checksum.json
               '';