diff options
Diffstat (limited to 'nixpkgs/pkgs/build-support/fetchurl')
| -rw-r--r-- | nixpkgs/pkgs/build-support/fetchurl/boot.nix | 25 | ||||
| -rw-r--r-- | nixpkgs/pkgs/build-support/fetchurl/builder.sh | 173 | ||||
| -rw-r--r-- | nixpkgs/pkgs/build-support/fetchurl/default.nix | 190 | ||||
| -rw-r--r-- | nixpkgs/pkgs/build-support/fetchurl/mirrors.nix | 416 | ||||
| -rw-r--r-- | nixpkgs/pkgs/build-support/fetchurl/tests.nix | 13 | ||||
| -rw-r--r-- | nixpkgs/pkgs/build-support/fetchurl/write-mirror-list.sh | 4 |
6 files changed, 821 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/build-support/fetchurl/boot.nix b/nixpkgs/pkgs/build-support/fetchurl/boot.nix new file mode 100644 index 000000000000..8f8c78b7a454 --- /dev/null +++ b/nixpkgs/pkgs/build-support/fetchurl/boot.nix @@ -0,0 +1,25 @@ +let mirrors = import ./mirrors.nix; in + +{ system }: + +{ url ? builtins.head urls +, urls ? [] +, sha256 ? "" +, hash ? "" +, name ? baseNameOf (toString url) +}: + +# assert exactly one hash is set +assert hash != "" || sha256 != ""; +assert hash != "" -> sha256 == ""; + +import <nix/fetchurl.nix> { + inherit system hash sha256 name; + + url = + # Handle mirror:// URIs. Since <nix/fetchurl.nix> currently + # supports only one URI, use the first listed mirror. + let m = builtins.match "mirror://([a-z]+)/(.*)" url; in + if m == null then url + else builtins.head (mirrors.${builtins.elemAt m 0}) + (builtins.elemAt m 1); +} diff --git a/nixpkgs/pkgs/build-support/fetchurl/builder.sh b/nixpkgs/pkgs/build-support/fetchurl/builder.sh new file mode 100644 index 000000000000..a82728ef1025 --- /dev/null +++ b/nixpkgs/pkgs/build-support/fetchurl/builder.sh @@ -0,0 +1,173 @@ +if [ -e "$NIX_ATTRS_SH_FILE" ]; then . "$NIX_ATTRS_SH_FILE"; elif [ -f .attrs.sh ]; then . .attrs.sh; fi +source $stdenv/setup + +source $mirrorsFile + +curlVersion=$(curl -V | head -1 | cut -d' ' -f2) + +# Curl flags to handle redirects, not use EPSV, handle cookies for +# servers to need them during redirects, and work on SSL without a +# certificate (this isn't a security problem because we check the +# cryptographic hash of the output anyway). +curl=( + curl + --location + --max-redirs 20 + --retry 3 + --disable-epsv + --cookie-jar cookies + --user-agent "curl/$curlVersion Nixpkgs/$nixpkgsVersion" +) + +if ! [ -f "$SSL_CERT_FILE" ]; then + curl+=(--insecure) +fi + +eval "curl+=($curlOptsList)" + +curl+=( + $curlOpts + $NIX_CURL_FLAGS +) + +downloadedFile="$out" +if [ -n "$downloadToTemp" ]; then downloadedFile="$TMPDIR/file"; fi + + +tryDownload() { + local url="$1" + echo + echo "trying $url" + local curlexit=18; + + success= + + # if we get error code 18, resume partial download + while [ $curlexit -eq 18 ]; do + # keep this inside an if statement, since on failure it doesn't abort the script + if "${curl[@]}" -C - --fail "$url" --output "$downloadedFile"; then + success=1 + break + else + curlexit=$?; + fi + done +} + + +finish() { + local skipPostFetch="$1" + + set +o noglob + + if [[ $executable == "1" ]]; then + chmod +x $downloadedFile + fi + + if [ -z "$skipPostFetch" ]; then + runHook postFetch + fi + + exit 0 +} + + +tryHashedMirrors() { + if test -n "$NIX_HASHED_MIRRORS"; then + hashedMirrors="$NIX_HASHED_MIRRORS" + fi + + for mirror in $hashedMirrors; do + url="$mirror/$outputHashAlgo/$outputHash" + if "${curl[@]}" --retry 0 --connect-timeout "${NIX_CONNECT_TIMEOUT:-15}" \ + --fail --silent --show-error --head "$url" \ + --write-out "%{http_code}" --output /dev/null > code 2> log; then + tryDownload "$url" + + # We skip postFetch here, because hashed-mirrors are + # already content addressed. So if $outputHash is in the + # hashed-mirror, changes from ‘postFetch’ would already be + # made. So, running postFetch will end up applying the + # change /again/, which we don’t want. + if test -n "$success"; then finish skipPostFetch; fi + else + # Be quiet about 404 errors, which we interpret as the file + # not being present on this particular mirror. + if test "$(cat code)" != 404; then + echo "error checking the existence of $url:" + cat log + fi + fi + done +} + + +# URL list may contain ?. No glob expansion for that, please +set -o noglob + +urls2= +for url in $urls; do + if test "${url:0:9}" != "mirror://"; then + urls2="$urls2 $url" + else + url2="${url:9}"; echo "${url2/\// }" > split; read site fileName < split + #varName="mirror_$site" + varName="$site" # !!! danger of name clash, fix this + if test -z "${!varName}"; then + echo "warning: unknown mirror:// site \`$site'" + else + mirrors=${!varName} + + # Allow command-line override by setting NIX_MIRRORS_$site. + varName="NIX_MIRRORS_$site" + if test -n "${!varName}"; then mirrors="${!varName}"; fi + + for url3 in $mirrors; do + urls2="$urls2 $url3$fileName"; + done + fi + fi +done +urls="$urls2" + +# Restore globbing settings +set +o noglob + +if test -n "$showURLs"; then + echo "$urls" > $out + exit 0 +fi + +if test -n "$preferHashedMirrors"; then + tryHashedMirrors +fi + +# URL list may contain ?. No glob expansion for that, please +set -o noglob + +success= +for url in $urls; do + if [ -z "$postFetch" ]; then + case "$url" in + https://github.com/*/archive/*) + echo "warning: archives from GitHub revisions should use fetchFromGitHub" + ;; + https://gitlab.com/*/-/archive/*) + echo "warning: archives from GitLab revisions should use fetchFromGitLab" + ;; + esac + fi + tryDownload "$url" + if test -n "$success"; then finish; fi +done + +# Restore globbing settings +set +o noglob + +if test -z "$preferHashedMirrors"; then + tryHashedMirrors +fi + + +echo "error: cannot download $name from any mirror" +exit 1 diff --git a/nixpkgs/pkgs/build-support/fetchurl/default.nix b/nixpkgs/pkgs/build-support/fetchurl/default.nix new file mode 100644 index 000000000000..a9c2c7c46d14 --- /dev/null +++ b/nixpkgs/pkgs/build-support/fetchurl/default.nix @@ -0,0 +1,190 @@ +{ lib, buildPackages ? { inherit stdenvNoCC; }, stdenvNoCC +, curl # Note that `curl' may be `null', in case of the native stdenvNoCC. +, cacert ? null }: + +let + + mirrors = import ./mirrors.nix; + + # Write the list of mirrors to a file that we can reuse between + # fetchurl instantiations, instead of passing the mirrors to + # fetchurl instantiations via environment variables. This makes the + # resulting store derivations (.drv files) much smaller, which in + # turn makes nix-env/nix-instantiate faster. + mirrorsFile = + buildPackages.stdenvNoCC.mkDerivation ({ + name = "mirrors-list"; + strictDeps = true; + builder = ./write-mirror-list.sh; + preferLocalBuild = true; + } // mirrors); + + # Names of the master sites that are mirrored (i.e., "sourceforge", + # "gnu", etc.). + sites = builtins.attrNames mirrors; + + impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ + # This variable allows the user to pass additional options to curl + "NIX_CURL_FLAGS" + + # This variable allows the user to override hashedMirrors from the + # command-line. + "NIX_HASHED_MIRRORS" + + # This variable allows overriding the timeout for connecting to + # the hashed mirrors. + "NIX_CONNECT_TIMEOUT" + ] ++ (map (site: "NIX_MIRRORS_${site}") sites); + +in + +{ # URL to fetch. + url ? "" + +, # Alternatively, a list of URLs specifying alternative download + # locations. They are tried in order. + urls ? [] + +, # Additional curl options needed for the download to succeed. + # Warning: Each space (no matter the escaping) will start a new argument. + # If you wish to pass arguments with spaces, use `curlOptsList` + curlOpts ? "" + +, # Additional curl options needed for the download to succeed. + curlOptsList ? [] + +, # Name of the file. If empty, use the basename of `url' (or of the + # first element of `urls'). + name ? "" + + # for versioned downloads optionally take pname + version. +, pname ? "" +, version ? "" + +, # SRI hash. + hash ? "" + +, # Legacy ways of specifying the hash. + outputHash ? "" +, outputHashAlgo ? "" +, sha1 ? "" +, sha256 ? "" +, sha512 ? "" + +, recursiveHash ? false + +, # Shell code to build a netrc file for BASIC auth + netrcPhase ? null + +, # Impure env vars (https://nixos.org/nix/manual/#sec-advanced-attributes) + # needed for netrcPhase + netrcImpureEnvVars ? [] + +, # Shell code executed after the file has been fetched + # successfully. This can do things like check or transform the file. + postFetch ? "" + +, # Whether to download to a temporary path rather than $out. Useful + # in conjunction with postFetch. The location of the temporary file + # is communicated to postFetch via $downloadedFile. + downloadToTemp ? false + +, # If true, set executable bit on downloaded file + executable ? false + +, # If set, don't download the file, but write a list of all possible + # URLs (resulting from resolving mirror:// URLs) to $out. + showURLs ? false + +, # Meta information, if any. + meta ? {} + + # Passthru information, if any. +, passthru ? {} + # Doing the download on a remote machine just duplicates network + # traffic, so don't do that by default +, preferLocalBuild ? true + + # Additional packages needed as part of a fetch +, nativeBuildInputs ? [ ] +}: + +let + urls_ = + if urls != [] && url == "" then + (if lib.isList urls then urls + else throw "`urls` is not a list") + else if urls == [] && url != "" then + (if lib.isString url then [url] + else throw "`url` is not a string") + else throw "fetchurl requires either `url` or `urls` to be set"; + + hash_ = + if with lib.lists; length (filter (s: s != "") [ hash outputHash sha1 sha256 sha512 ]) > 1 + then throw "multiple hashes passed to fetchurl" else + + if hash != "" then { outputHashAlgo = null; outputHash = hash; } + else if outputHash != "" then + if outputHashAlgo != "" then { inherit outputHashAlgo outputHash; } + else throw "fetchurl was passed outputHash without outputHashAlgo" + else if sha512 != "" then { outputHashAlgo = "sha512"; outputHash = sha512; } + else if sha256 != "" then { outputHashAlgo = "sha256"; outputHash = sha256; } + else if sha1 != "" then { outputHashAlgo = "sha1"; outputHash = sha1; } + else if cacert != null then { outputHashAlgo = "sha256"; outputHash = ""; } + else throw "fetchurl requires a hash for fixed-output derivation: ${lib.concatStringsSep ", " urls_}"; +in + +assert (lib.isList curlOpts) -> lib.warn '' + fetchurl for ${toString (builtins.head urls_)}: curlOpts is a list (${lib.generators.toPretty { multiline = false; } curlOpts}), which is not supported anymore. + - If you wish to get the same effect as before, for elements with spaces (even if escaped) to expand to multiple curl arguments, use a string argument instead: + curlOpts = ${lib.strings.escapeNixString (toString curlOpts)}; + - If you wish for each list element to be passed as a separate curl argument, allowing arguments to contain spaces, use curlOptsList instead: + curlOptsList = [ ${lib.concatMapStringsSep " " lib.strings.escapeNixString curlOpts} ];'' true; + +stdenvNoCC.mkDerivation (( + if (pname != "" && version != "") then + { inherit pname version; } + else + { name = + if showURLs then "urls" + else if name != "" then name + else baseNameOf (toString (builtins.head urls_)); + } +) // { + builder = ./builder.sh; + + nativeBuildInputs = [ curl ] ++ nativeBuildInputs; + + urls = urls_; + + # If set, prefer the content-addressable mirrors + # (http://tarballs.nixos.org) over the original URLs. + preferHashedMirrors = true; + + # New-style output content requirements. + inherit (hash_) outputHashAlgo outputHash; + + SSL_CERT_FILE = if (hash_.outputHash == "" || hash_.outputHash == lib.fakeSha256 || hash_.outputHash == lib.fakeSha512 || hash_.outputHash == lib.fakeHash) + then "${cacert}/etc/ssl/certs/ca-bundle.crt" + else "/no-cert-file.crt"; + + outputHashMode = if (recursiveHash || executable) then "recursive" else "flat"; + + inherit curlOpts; + curlOptsList = lib.escapeShellArgs curlOptsList; + inherit showURLs mirrorsFile postFetch downloadToTemp executable; + + impureEnvVars = impureEnvVars ++ netrcImpureEnvVars; + + nixpkgsVersion = lib.trivial.release; + + inherit preferLocalBuild; + + postHook = if netrcPhase == null then null else '' + ${netrcPhase} + curlOpts="$curlOpts --netrc-file $PWD/netrc" + ''; + + inherit meta; + passthru = { inherit url; } // passthru; +}) diff --git a/nixpkgs/pkgs/build-support/fetchurl/mirrors.nix b/nixpkgs/pkgs/build-support/fetchurl/mirrors.nix new file mode 100644 index 000000000000..af0468b3e494 --- /dev/null +++ b/nixpkgs/pkgs/build-support/fetchurl/mirrors.nix @@ -0,0 +1,416 @@ +{ + + # Content-addressable Nix mirrors + hashedMirrors = [ + "https://tarballs.nixos.org" + ]; + + # Mirrors for mirror://site/filename URIs, where "site" is + # "sourceforge", "gnu", etc. + + # Alsa Project + alsa = [ + "https://www.alsa-project.org/files/pub/" + "ftp://ftp.alsa-project.org/pub/" + "http://alsa.cybermirror.org/" + "http://www.mirrorservice.org/sites/ftp.alsa-project.org/pub/" + ]; + + # Apache + apache = [ + "https://dlcdn.apache.org/" + "https://www-eu.apache.org/dist/" + "https://ftp.wayne.edu/apache/" + "https://www.apache.org/dist/" + "https://archive.apache.org/dist/" # fallback for old releases + "https://apache.cs.uu.nl/" + "https://apache.cs.utah.edu/" + "http://ftp.tudelft.nl/apache/" + "ftp://ftp.funet.fi/pub/mirrors/apache.org/" + ]; + + # Bioconductor mirrors (from https://bioconductor.org/about/mirrors/) + # The commented-out ones don't seem to allow direct package downloads; + # they serve error messages that result in hash mismatches instead + bioc = [ + # http://bioc.ism.ac.jp/ + # http://bioc.openanalytics.eu/ + # http://bioconductor.fmrp.usp.br/ + # http://mirror.aarnet.edu.au/pub/bioconductor/ + # http://watson.nci.nih.gov/bioc_mirror/ + "https://bioconductor.statistik.tu-dortmund.de/packages/" + "https://mirrors.ustc.edu.cn/bioc/" + "http://bioconductor.jp/packages/" + ]; + + # CRAN mirrors + cran = [ + "https://cran.r-project.org/src/contrib/" + ]; + + # BitlBee mirrors, see https://www.bitlbee.org/main.php/mirrors.html + bitlbee = [ + "https://get.bitlbee.org/" + "https://ftp.snt.utwente.nl/pub/software/bitlbee/" + "http://bitlbee.intergenia.de/" + ]; + + # GCC + gcc = [ + "https://mirror.koddos.net/gcc/" + "https://bigsearcher.com/mirrors/gcc/" + "ftp://ftp.nluug.nl/mirror/languages/gcc/" + "ftp://ftp.fu-berlin.de/unix/languages/gcc/" + "ftp://ftp.irisa.fr/pub/mirrors/gcc.gnu.org/gcc/" + "ftp://gcc.gnu.org/pub/gcc/" + ]; + + # GNOME + gnome = [ + # This one redirects to some mirror closeby, so it should be all you need + "https://download.gnome.org/" + + "https://fr2.rpmfind.net/linux/gnome.org/" + "https://ftp.acc.umu.se/pub/GNOME/" + "https://ftp.belnet.be/mirror/ftp.gnome.org/" + "ftp://ftp.cse.buffalo.edu/pub/Gnome/" + "ftp://ftp.nara.wide.ad.jp/pub/X11/GNOME/" + ]; + + # GNU (https://www.gnu.org/prep/ftp.html) + gnu = [ + # This one redirects to a (supposedly) nearby and (supposedly) up-to-date + # mirror + "https://ftpmirror.gnu.org/" + + "https://ftp.nluug.nl/pub/gnu/" + "https://mirrors.kernel.org/gnu/" + "https://mirror.ibcp.fr/pub/gnu/" + "https://mirror.dogado.de/gnu/" + "https://mirror.tochlab.net/pub/gnu/" + + # This one is the master repository, and thus it's always up-to-date + "https://ftp.gnu.org/pub/gnu/" + + "ftp://ftp.funet.fi/pub/mirrors/ftp.gnu.org/gnu/" + ]; + + # GnuPG + gnupg = [ + "https://gnupg.org/ftp/gcrypt/" + "https://mirrors.dotsrc.org/gcrypt/" + "https://ftp.heanet.ie/mirrors/ftp.gnupg.org/gcrypt/" + "https://www.mirrorservice.org/sites/ftp.gnupg.org/gcrypt/" + "http://www.ring.gr.jp/pub/net/" + ]; + + # IBiblio (former metalab/sunsite) + # Most of the time the expressions refer to the /pub/Linux/ subdirectory; + # however there are other useful files outside it + ibiblioPubLinux = [ + "https://www.ibiblio.org/pub/Linux/" + "ftp://ftp.ibiblio.org/pub/linux/" + "ftp://ftp.gwdg.de/pub/linux/metalab/" + "ftp://ftp.metalab.unc.edu/pub/linux/" + ]; + + # ImageMagick mirrors, see https://www.imagemagick.org/script/mirror.php + imagemagick = [ + "https://www.imagemagick.org/download/" + "https://mirror.checkdomain.de/imagemagick/" + "https://ftp.nluug.nl/ImageMagick/" + "https://ftp.sunet.se/mirror/imagemagick.org/ftp/" + "ftp://ftp.sunet.se/mirror/imagemagick.org/ftp/" # also contains older versions removed from most mirrors + ]; + + # Mirrors from https://download.kde.org/ls-lR.mirrorlist + kde = [ + "https://cdn.download.kde.org/" + "https://download.kde.org/download.php?url=" + "https://ftp.gwdg.de/pub/linux/kde/" + "https://mirrors.ocf.berkeley.edu/kde/" + "https://mirrors.mit.edu/kde/" + "https://mirrors.ustc.edu.cn/kde/" + "https://ftp.funet.fi/pub/mirrors/ftp.kde.org/pub/kde/" + ]; + + # kernel.org's /pub (/pub/{linux,software}) tree + kernel = [ + "https://cdn.kernel.org/pub/" + "http://linux-kernel.uio.no/pub/" + "ftp://ftp.funet.fi/pub/mirrors/ftp.kernel.org/pub/" + ]; + + # MySQL + mysql = [ + "https://cdn.mysql.com/Downloads/" + ]; + + # Maven Central + maven = [ + "https://repo1.maven.org/maven2/" + ]; + + # Mozilla projects + mozilla = [ + "https://download.cdn.mozilla.net/pub/mozilla.org/" + "https://archive.mozilla.org/pub/" + ]; + + # OSDN (formerly SourceForge.jp) + osdn = [ + "https://osdn.dl.osdn.jp/" + "https://osdn.mirror.constant.com/" + "https://mirrors.gigenet.com/OSDN/" + "https://osdn.dl.sourceforge.jp/" + "https://jaist.dl.sourceforge.jp/" + ]; + + # PostgreSQL + postgresql = [ + "https://ftp.postgresql.org/pub/" + ]; + + # Qt + qt = [ + "https://download.qt.io/" + ]; + + # Sage mirrors (https://www.sagemath.org/mirrors.html) + sageupstream = [ + # Africa (HTTPS) + "https://sagemath.mirror.ac.za/spkg/upstream/" + "https://mirror.ufs.ac.za/sagemath/spkg/upstream/" + + # America, North (HTTPS) + "https://mirrors.mit.edu/sage/spkg/upstream/" + "https://mirrors.xmission.com/sage/spkg/upstream/" + + # Asia (HTTPS) + "https://mirrors.tuna.tsinghua.edu.cn/sagemath/spkg/upstream/" + "https://mirrors.ustc.edu.cn/sagemath/spkg/upstream/" + "http://ftp.tsukuba.wide.ad.jp/software/sage/spkg/upstream/" + "https://ftp.yz.yamagata-u.ac.jp/pub/math/sage/spkg/upstream/" + "https://mirror.yandex.ru/mirrors/sage.math.washington.edu/spkg/upstream/" + + # Australia (HTTPS) + "https://mirror.aarnet.edu.au/pub/sage/spkg/upstream/" + + # Europe (HTTPS) + "https://sage.mirror.garr.it/mirrors/sage/spkg/upstream/" + "https://www-ftp.lip6.fr/pub/math/sagemath/spkg/upstream/" + + # Africa (non-HTTPS) + "ftp://ftp.sun.ac.za/pub/mirrors/www.sagemath.org/spkg/upstream/" + + # America, North (non-HTTPS) + "http://www.cecm.sfu.ca/sage/spkg/upstream/" + + # America, South (non-HTTPS) + "http://sagemath.c3sl.ufpr.br/spkg/upstream/" + "http://linorg.usp.br/sage/spkg/upstream" + + # Asia (non-HTTPS) + "http://ftp.kaist.ac.kr/sage/spkg/upstream/" + "http://ftp.riken.jp/sagemath/spkg/upstream/" + + # Europe (non-HTTPS) + "http://mirrors.fe.up.pt/pub/sage/spkg/upstream/" + "http://ftp.ntua.gr/pub/sagemath/spkg/upstream/" + ]; + + # SAMBA + samba = [ + "https://www.samba.org/ftp/" + "http://www.samba.org/ftp/" + ]; + + # GNU Savannah + savannah = [ + # Mirrors from https://download-mirror.savannah.gnu.org/releases/00_MIRRORS.html + "https://mirror.easyname.at/nongnu/" + "https://savannah.c3sl.ufpr.br/" + "https://mirror.csclub.uwaterloo.ca/nongnu/" + "https://mirror.cedia.org.ec/nongnu/" + "https://ftp.igh.cnrs.fr/pub/nongnu/" + "https://mirror6.layerjet.com/nongnu" + "https://mirror.netcologne.de/savannah/" + "https://ftp.cc.uoc.gr/mirrors/nongnu.org/" + "https://nongnu.uib.no/" + "https://ftp.acc.umu.se/mirror/gnu.org/savannah/" + "http://mirror2.klaus-uwe.me/nongnu/" + "http://mirrors.fe.up.pt/pub/nongnu/" + "http://ftp.twaren.net/Unix/NonGNU/" + "http://savannah-nongnu-org.ip-connect.vn.ua/" + "http://www.mirrorservice.org/sites/download.savannah.gnu.org/releases/" + "http://gnu.mirrors.pair.com/savannah/savannah/" + "ftp://mirror.easyname.at/nongnu/" + "ftp://mirror2.klaus-uwe.me/nongnu/" + "ftp://mirror.csclub.uwaterloo.ca/nongnu/" + "ftp://ftp.igh.cnrs.fr/pub/nongnu/" + "ftp://mirror.netcologne.de/savannah/" + "ftp://nongnu.uib.no/pub/nongnu/" + "ftp://mirrors.fe.up.pt/pub/nongnu/" + "ftp://ftp.twaren.net/Unix/NonGNU/" + "ftp://savannah-nongnu-org.ip-connect.vn.ua/mirror/savannah.nongnu.org/" + "ftp://ftp.mirrorservice.org/sites/download.savannah.gnu.org/releases/" + ]; + + # SourceForge + sourceforge = [ + "https://downloads.sourceforge.net/" + "https://prdownloads.sourceforge.net/" + "https://netcologne.dl.sourceforge.net/sourceforge/" + "https://versaweb.dl.sourceforge.net/sourceforge/" + "https://freefr.dl.sourceforge.net/sourceforge/" + "https://osdn.dl.sourceforge.net/sourceforge/" + ]; + + # Steam Runtime + steamrt = [ + "https://repo.steampowered.com/steamrt/" + "https://public.abbradar.moe/steamrt/" + ]; + + # TCSH shell + tcsh = [ + "https://astron.com/pub/tcsh/" + "https://astron.com/pub/tcsh/old/" + "http://ftp.funet.fi/pub/mirrors/ftp.astron.com/pub/tcsh/" + "http://ftp.funet.fi/pub/mirrors/ftp.astron.com/pub/tcsh/old/" + "ftp://ftp.astron.com/pub/tcsh/" + "ftp://ftp.astron.com/pub/tcsh/old/" + "ftp://ftp.funet.fi/pub/unix/shells/tcsh/" + "ftp://ftp.funet.fi/pub/unix/shells/tcsh/old/" + ]; + + # XFCE + xfce = [ + "https://archive.xfce.org/" + "https://mirror.netcologne.de/xfce/" + "https://archive.be.xfce.org/xfce/" + "https://archive.al-us.xfce.org/" + "http://archive.se.xfce.org/xfce/" + "http://mirror.perldude.de/archive.xfce.org/" + "http://archive.be2.xfce.org/" + "http://ftp.udc.es/xfce/" + ]; + + # X.org + xorg = [ + "https://xorg.freedesktop.org/releases/" + "https://ftp.x.org/archive/" + ]; + + ### Programming languages' package repos + + # Perl CPAN + cpan = [ + "https://cpan.metacpan.org/" + "https://cpan.perl.org/" + "https://mirrors.kernel.org/CPAN/" + "https://backpan.perl.org/" # for old releases + ]; + + # Haskell Hackage + hackage = [ + "https://hackage.haskell.org/package/" + ]; + + # Lua Rocks + luarocks = [ + "https://luarocks.org/" + "https://raw.githubusercontent.com/rocks-moonscript-org/moonrocks-mirror/master/" + "https://luafr.org/moonrocks/" + ]; + + # Python PyPI + pypi = [ + "https://files.pythonhosted.org/packages/source/" + # pypi.io is a more semantic link, but atm it’s referencing + # files.pythonhosted.org over two redirects + "https://pypi.io/packages/source/" + ]; + + # Python Test-PyPI + testpypi = [ + "https://test.pypi.io/packages/source/" + ]; + + ### Linux distros + + # CentOS + centos = [ + # For old releases + "https://vault.centos.org/" + "https://archive.kernel.org/centos-vault/" + "https://ftp.jaist.ac.jp/pub/Linux/CentOS-vault/" + "https://mirrors.aliyun.com/centos-vault/" + "https://mirror.chpc.utah.edu/pub/vault.centos.org/" + "https://mirror.math.princeton.edu/pub/centos-vault/" + "https://mirrors.tripadvisor.com/centos-vault/" + "http://mirror.centos.org/centos/" + ]; + + # Debian + debian = [ + "https://httpredir.debian.org/debian/" + "https://ftp.debian.org/debian/" + "https://mirrors.edge.kernel.org/debian/" + "ftp://ftp.de.debian.org/debian/" + "ftp://ftp.fr.debian.org/debian/" + "ftp://ftp.nl.debian.org/debian/" + "ftp://ftp.ru.debian.org/debian/" + "http://archive.debian.org/debian-archive/debian/" + "ftp://ftp.funet.fi/pub/mirrors/ftp.debian.org/debian/" + ]; + + # Fedora + # Please add only full mirrors that carry old Fedora distributions as well + # See: https://mirrors.fedoraproject.org/publiclist (but not all carry old content) + fedora = [ + "https://archives.fedoraproject.org/pub/fedora/" + "https://fedora.osuosl.org/" + "https://ftp.funet.fi/pub/mirrors/ftp.redhat.com/pub/fedora/" + "https://ftp.linux.cz/pub/linux/fedora/" + "https://archives.fedoraproject.org/pub/archive/fedora/" + "http://ftp.nluug.nl/pub/os/Linux/distr/fedora/" + "http://mirror.csclub.uwaterloo.ca/fedora/" + "http://mirror.1000mbps.com/fedora/" + ]; + + # Gentoo + gentoo = [ + "https://ftp.snt.utwente.nl/pub/os/linux/gentoo/" + "https://distfiles.gentoo.org/" + "https://mirrors.kernel.org/gentoo/" + ]; + + # openSUSE + opensuse = [ + "https://opensuse.hro.nl/opensuse/distribution/" + "https://ftp.funet.fi/pub/linux/mirrors/opensuse/distribution/" + "https://ftp.opensuse.org/pub/opensuse/distribution/" + "https://ftp5.gwdg.de/pub/opensuse/discontinued/distribution/" + "https://mirrors.edge.kernel.org/opensuse/distribution/" + "http://ftp.hosteurope.de/mirror/ftp.opensuse.org/discontinued/" + ]; + + # Ubuntu + ubuntu = [ + "https://nl.archive.ubuntu.com/ubuntu/" + "https://old-releases.ubuntu.com/ubuntu/" + "https://mirrors.edge.kernel.org/ubuntu/" + "http://de.archive.ubuntu.com/ubuntu/" + "http://archive.ubuntu.com/ubuntu/" + ]; + + # ... and other OSes in general + + # OpenBSD + openbsd = [ + "https://ftp.openbsd.org/pub/OpenBSD/" + "ftp://ftp.nluug.nl/pub/OpenBSD/" + "ftp://ftp-stud.fht-esslingen.de/pub/OpenBSD/" + ]; +} diff --git a/nixpkgs/pkgs/build-support/fetchurl/tests.nix b/nixpkgs/pkgs/build-support/fetchurl/tests.nix new file mode 100644 index 000000000000..e348d77db0bd --- /dev/null +++ b/nixpkgs/pkgs/build-support/fetchurl/tests.nix @@ -0,0 +1,13 @@ +{ testers, fetchurl, jq, moreutils, ... }: { + # Tests that we can send custom headers with spaces in them + header = + let headerValue = "Test '\" <- These are some quotes"; + in testers.invalidateFetcherByDrvHash fetchurl { + url = "https://httpbin.org/headers"; + sha256 = builtins.hashString "sha256" (headerValue + "\n"); + curlOptsList = [ "-H" "Hello: ${headerValue}" ]; + postFetch = '' + ${jq}/bin/jq -r '.headers.Hello' $out | ${moreutils}/bin/sponge $out + ''; + }; +} diff --git a/nixpkgs/pkgs/build-support/fetchurl/write-mirror-list.sh b/nixpkgs/pkgs/build-support/fetchurl/write-mirror-list.sh new file mode 100644 index 000000000000..2dabd2e722be --- /dev/null +++ b/nixpkgs/pkgs/build-support/fetchurl/write-mirror-list.sh @@ -0,0 +1,4 @@ +source $stdenv/setup + +# !!! this is kinda hacky. +set | grep -E '^[a-zA-Z]+=.*://' > $out |