Merge commit '1a57d96edd156958b12782e8c8b6a374142a7248'

3 files changed, 161 insertions, 68 deletions

diff --git a/nixpkgs/pkgs/common-updater/scripts/mark-broken b/nixpkgs/pkgs/common-updater/scripts/mark-broken
index d128d0d458ba..f132c86c5e8b 100755
--- a/nixpkgs/pkgs/common-updater/scripts/mark-broken
+++ b/nixpkgs/pkgs/common-updater/scripts/mark-broken
@@ -1,86 +1,106 @@
 #!/usr/bin/env bash
-set -e
+
+# This script is meant to be used to mark failing hydra builds as broken in the meta attrs
+# To use the script, you should pass the list of failing attrs as arguments to the script.
+#
+# Example: `cat failing-attrs | xargs ./pkgs/common-updater/scripts/mark-broken`
+#
+# Generating a list of failing attrs: (this should be improved at a later date)
+#   - Go to the most recent hydra evaluation with all builds completed
+#   - Select the "builds still failing" tab
+#   - Highlight and select all packages, should be prefixed with `nixpkgs.`
+#   - Use regex and editor foo to leave only the attr names
+#   - Use the above example command to then execute the script
+#
+# OTHER NOTES:
+#   - The `denyFileList` and `denyAttrList` will likely need to be updated slightly
+#     to align with the conventions used in nixpkgs at execution time
+#   - Any attrs which failed for any reason will be written to `failed-marks.txt`.
+#     Those attrs will likely need manual attention as disablement will likely be conditional.
 
 scriptName=mark-broken # do not use the .wrapped name
 
-die() {
-    echo "$scriptName: error: $1" >&2
-    exit 1
+failMark() {
+        local attr=$1
+        shift 1
+
+        echo "$attr: $@" >&2
+        echo $attr >> failed-marks.txt
 }
 
 usage() {
-    echo "Usage: $scriptName <attr> [--new-value=<new-value>]"
+    echo "Usage: $scriptName <attrs>"
 }
 
-args=()
-
-for arg in "$@"; do
-    case $arg in
-        --new-value=*)
-            newValue="${arg#*=}"
-        ;;
-        --help)
-            usage
-            exit 0
-        ;;
-        --*)
-            echo "$scriptName: Unknown argument: $arg"
-            usage
-            exit 1
-        ;;
-        *)
-            args["${#args[*]}"]=$arg
-        ;;
-    esac
-done
-
-attr=${args[0]}
-
-if (( "${#args[*]}" < 1 )); then
+if (( "${#@}" < 1 )); then
     echo "$scriptName: Too few arguments"
     usage
     exit 1
 fi
 
-if (( "${#args[*]}" > 1 )); then
-    echo "$scriptName: Too many arguments"
-    usage
-    exit 1
-fi
+# in case we resolve to an auto-generated file, just skip these entries
+denyFileList=(
+        node-packages.nix # node, it will mark all node packages as broken
+        generic-builder.nix # haskell, it will mark all haskell packages as broken
+)
 
-if [ -z $newValue ]; then
-  newValue="true"
-fi
+# ignore older versions of parameterized packages sets, these likely need
+# to be conditionally disabled
+denyAttrList=(
+        python27Packages
+        python37Packages
+        libsForQt512
+        linuxPackages_
+        rubyPackages_
+)
 
-nixFile=$(nix-instantiate --eval --json -E "with import ./. {}; (builtins.unsafeGetAttrPos \"description\" $attr.meta).file" | jq -r .)
-if [[ ! -f "$nixFile" ]]; then
-    die "Couldn't evaluate 'builtins.unsafeGetAttrPos \"description\" $attr.meta' to locate the .nix file!"
-fi
+function attemptToMarkBroken() {
+        local attr=$1
 
-# Insert broken attribute
-sed -i.bak "$nixFile" -r \
-  -e "/^\s*broken\s*=.*$/d" \
-  -e "s/(\s*)meta\s*=.*\{/&\n\1  broken = $newValue;/"
+        # skip likely to be noisy attrs
+        for badAttr in ${denyAttrList[@]};do
+                if [[ $attr =~ $badAttr ]]; then
+                        failMark $attr "attr contained $badAttr, skipped."
+                        return
+                fi
+        done
 
-if cmp -s "$nixFile" "$nixFile.bak"; then
-    mv "$nixFile.bak" "$nixFile"
-    die "Failed to mark the package as broken! Does it have a meta attribute?"
-fi
+        nixFile=$(nix-instantiate --eval --json -E "with import ./. {}; (builtins.unsafeGetAttrPos \"description\" $attr.meta).file" 2>/dev/null | jq -r .)
+        if [[ ! -f "$nixFile" ]]; then
+            failMark $attr "Couldn't locate correct file"
+            return
+        fi
 
-if [[ "$newValue" == "true" ]]; then
-    # broken should evaluate to true in any case now
-    markedSuccessfully=$(nix-instantiate --eval -E "with import ./. {}; $attr.meta.broken" || true)
-    if [[ ! "$markedSuccessfully" == "true" ]]; then
-        mv "$nixFile.bak" "$nixFile"
-        die "Couldn't verify the broken attribute to be set correctly, restoring backup!"
-    fi
-else
-    # we can not check if broken evaluates to the correct value, but we can check that it does evaluate
-    if ! nix-instantiate --eval -E "with import ./. {}; $attr.meta.broken" >/dev/null; then
-        mv "$nixFile.bak" "$nixFile"
-        die "Couldn't verify the broken attribute to be set correctly, restoring backup!"
-    fi
-fi
+        # skip files which are auto-generated
+        for filename in ${denyFileList[@]};do
+                if [[ "$filename" == $(basename $nixFile) ]]; then
+                        failMark $attr "filename matched $filename, skipped."
+                        return
+                fi
+        done
+
+        # Insert broken attribute
+        sed -i.bak "$nixFile" -r \
+          -e "/^\s*broken\s*=.*$/d" \
+          -e "s/(\s*)meta\s*=.*\{/&\n\1  broken = true;/"
+
+        if cmp -s "$nixFile" "$nixFile.bak"; then
+            mv "$nixFile.bak" "$nixFile"
+            failMark $attr "Does it have a meta attribute?"
+            return
+        fi
 
-rm -f "$nixFile.bak"
-rm -f "$attr.fetchlog"
+        # broken should evaluate to true in any case now
+        markedSuccessfully=$(nix-instantiate --eval -E "with import ./. {}; $attr.meta.broken")
+        if [[ "$markedSuccessfully" != "true" ]]; then
+            mv "$nixFile.bak" "$nixFile"
+            failMark $attr "$attr.meta.broken doesn't evaluate to true."
+            return
+        fi
+
+        rm -f "$nixFile.bak"
+}
+
+for attr in $@; do
+        attemptToMarkBroken $attr
+done
diff --git a/nixpkgs/pkgs/common-updater/scripts/update-source-version b/nixpkgs/pkgs/common-updater/scripts/update-source-version
index 6a66f94597f4..181561242fc3 100755
--- a/nixpkgs/pkgs/common-updater/scripts/update-source-version
+++ b/nixpkgs/pkgs/common-updater/scripts/update-source-version
@@ -10,8 +10,8 @@ die() {
 
 usage() {
     echo "Usage: $scriptName <attr> <version> [<new-source-hash>] [<new-source-url>]"
-    echo "                              [--version-key=<version-key>] [--system=<system>] [--file=<file-to-update>]"
-    echo "                              [--ignore-same-hash]"
+    echo "                              [--version-key=<version-key>] [--system=<system>] [--file=<file-to-update>] [--rev=<revision>]"
+    echo "                              [--ignore-same-hash] [--print-changes]"
 }
 
 args=()
@@ -30,9 +30,15 @@ for arg in "$@"; do
                 die "Could not find provided file $nixFile"
             fi
         ;;
+        --rev=*)
+            newRevision="${arg#*=}"
+        ;;
         --ignore-same-hash)
             ignoreSameHash="true"
         ;;
+        --print-changes)
+            printChanges="true"
+        ;;
         --help)
             usage
             exit 0
@@ -102,9 +108,19 @@ fi
 
 if [[ "$oldVersion" = "$newVersion" ]]; then
     echo "$scriptName: New version same as old version, nothing to do." >&2
+    if [ -n "$printChanges" ]; then
+        printf '[]\n'
+    fi
     exit 0
 fi
 
+if [[ -n "$newRevision" ]]; then
+    oldRevision=$(nix-instantiate $systemArg --eval -E "with import ./. {}; $attr.src.rev" | tr -d '"')
+    if [[ -z "$oldRevision" ]]; then
+        die "Couldn't evaluate source revision from '$attr.src'!"
+    fi
+fi
+
 # Escape regex metacharacter that are allowed in store path names
 oldVersionEscaped=$(echo "$oldVersion" | sed -re 's|[.+]|\\&|g')
 oldUrlEscaped=$(echo "$oldUrl" | sed -re 's|[${}.+]|\\&|g')
@@ -168,6 +184,15 @@ if cmp -s "$nixFile" "$nixFile.bak"; then
     die "Failed to replace source hash of '$attr' to a temporary hash!"
 fi
 
+# Replace new revision, if given
+if [[ -n "$newRevision" ]]; then
+    sed -i "$nixFile" -re "s|\"$oldRevision\"|\"$newRevision\"|"
+
+    if cmp -s "$nixFile" "$nixFile.bak"; then
+        die "Failed to replace source revision '$oldRevision' to '$newRevision' in '$attr'!"
+    fi
+fi
+
 # If new hash not given on the command line, recalculate it ourselves.
 if [[ -z "$newHash" ]]; then
     nix-build $systemArg --no-out-link -A "$attr.src" 2>"$attr.fetchlog" >/dev/null || true
@@ -197,3 +222,7 @@ fi
 
 rm -f "$nixFile.bak"
 rm -f "$attr.fetchlog"
+
+if [ -n "$printChanges" ]; then
+    printf '[{"attrPath":"%s","oldVersion":"%s","newVersion":"%s","files":["%s"]}]\n' "$attr" "$oldVersion" "$newVersion" "$nixFile"
+fi
diff --git a/nixpkgs/pkgs/common-updater/unstable-updater.nix b/nixpkgs/pkgs/common-updater/unstable-updater.nix
new file mode 100644
index 000000000000..94cd33b9a26b
--- /dev/null
+++ b/nixpkgs/pkgs/common-updater/unstable-updater.nix
@@ -0,0 +1,44 @@
+{ stdenv
+, writeShellScript
+, coreutils
+, git
+, nix
+, common-updater-scripts
+}:
+
+# This is an updater for unstable packages that should always use the latest
+# commit.
+{ url ? null # The git url, if empty it will be set to src.url
+}:
+
+let
+  updateScript = writeShellScript "unstable-update-script.sh" ''
+    set -ex
+
+    url="$1"
+
+    # By default we set url to src.url
+    if [[ -z "$url" ]]; then
+        url="$(${nix}/bin/nix-instantiate $systemArg --eval -E \
+                   "with import ./. {}; $UPDATE_NIX_ATTR_PATH.src.url" \
+            | tr -d '"')"
+    fi
+
+    # Get info about HEAD from a shallow git clone
+    tmpdir="$(${coreutils}/bin/mktemp -d)"
+    ${git}/bin/git clone --bare --depth=1 "$url" "$tmpdir"
+    pushd "$tmpdir"
+    commit_date="$(${git}/bin/git show -s --pretty='format:%cs')"
+    commit_sha="$(${git}/bin/git show -s --pretty='format:%H')"
+    popd
+    ${coreutils}/bin/rm -rf "$tmpdir"
+
+    # update the nix expression
+    ${common-updater-scripts}/bin/update-source-version \
+        "$UPDATE_NIX_ATTR_PATH" \
+        "unstable-$commit_date" \
+        --rev="$commit_sha"
+  '';
+
+in [ updateScript url ]
+