diff options
Diffstat (limited to 'pkgs/tools/security/semgrep')
| -rw-r--r-- | pkgs/tools/security/semgrep/common.nix | 55 | ||||
| -rw-r--r-- | pkgs/tools/security/semgrep/default.nix | 17 | ||||
| -rw-r--r-- | pkgs/tools/security/semgrep/semgrep-core.nix | 39 | ||||
| -rwxr-xr-x | pkgs/tools/security/semgrep/update.sh | 60 |
4 files changed, 102 insertions, 69 deletions
diff --git a/pkgs/tools/security/semgrep/common.nix b/pkgs/tools/security/semgrep/common.nix index 48381e9eb0e6..0ad680b0ddee 100644 --- a/pkgs/tools/security/semgrep/common.nix +++ b/pkgs/tools/security/semgrep/common.nix @@ -1,54 +1,39 @@ -{ lib, fetchFromGitHub, fetchzip, stdenv }: +{ lib }: rec { - version = "1.15.0"; + version = "1.27.0"; - src = fetchFromGitHub { - owner = "returntocorp"; - repo = "semgrep"; - rev = "v${version}"; - sha256 = "sha256-x+AOt6nn2hN4MODFZCvlq0kZ3VLoS7rVcFGGCEssIu0="; - }; + srcHash = "sha256-F6n3LQY4a5sO6c8SMQF9YjjgOS+v2SH+UQPwhg2EX7Q="; # submodule dependencies # these are fetched so we: # 1. don't fetch the many submodules we don't need # 2. avoid fetchSubmodules since it's prone to impurities submodules = { - "cli/src/semgrep/lang" = fetchFromGitHub { - owner = "returntocorp"; - repo = "semgrep-langs"; - rev = "08656cdefc9e6818c64e168cf51ee1e76ea8829e"; - sha256 = "sha256-vYf33JhfvEDmt/VW0hBOmqailIERS0GdUgrPuCxWt9I="; - }; - "cli/src/semgrep/semgrep_interfaces" = fetchFromGitHub { + "cli/src/semgrep/semgrep_interfaces" = { owner = "returntocorp"; repo = "semgrep-interfaces"; - rev = "ba9241ca8f13dea72a4ca5c5eae99f45c071c8b4"; - sha256 = "sha256-2rcMmN42445AivcyYLPeE+HBYOyxJijQME1UUr9HISA="; + rev = "213f67abea73546ca6111e1bbf0ef96aa917c940"; + hash = "sha256-HeNHJkTje9j16+dwsfyMhoqQn/J18q/7XvQPRwgTw/Y="; }; }; # fetch pre-built semgrep-core since the ocaml build is complex and relies on # the opam package manager at some point - core = rec { - data = { - x86_64-linux = { - suffix = "-ubuntu-16.04.tgz"; - sha256 = "sha256-vLtV1WAnOD6HhgrWYIP0NfXHKfvXORksdNp5UTG1QWc="; - }; - x86_64-darwin = { - suffix = "-osx.zip"; - sha256 = "sha256-6+ENjOOIJ5TSjpnJ5pDudblrWj/FLUe66UGr6V9c0HQ="; - }; + # pulling it out of the python wheel as r2c no longer release a built binary + # on github releases + core = { + x86_64-linux = { + platform = "any"; + hash = "sha256-cRj81dXpAE6S0EXajsRikOIAPzlUf42FhiDCWjv+wZQ="; + }; + x86_64-darwin = { + platform = "macosx_10_14_x86_64"; + hash = "sha256-jqfGVZGF/DFgXkr7kQg6QyqEELSr8AKE3Ga8kTftnIY="; }; - src = let - inherit (stdenv.hostPlatform) system; - selectSystemData = data: data.${system} or (throw "Unsupported system: ${system}"); - inherit (selectSystemData data) suffix sha256; - in fetchzip { - url = "https://github.com/returntocorp/semgrep/releases/download/v${version}/semgrep-v${version}${suffix}"; - inherit sha256; + aarch64-darwin = { + platform = "macosx_11_0_arm64"; + hash = "sha256-e/uCSRMdbVD0lvc0hukbiUzheqRNIIh1LgMq6Ae7JYI="; }; }; @@ -66,7 +51,5 @@ rec { ''; license = licenses.lgpl21Plus; maintainers = with maintainers; [ jk ambroisie ]; - # limited by semgrep-core - platforms = [ "x86_64-linux" "x86_64-darwin" ]; }; } diff --git a/pkgs/tools/security/semgrep/default.nix b/pkgs/tools/security/semgrep/default.nix index 0e9c3ddc391c..23749643a443 100644 --- a/pkgs/tools/security/semgrep/default.nix +++ b/pkgs/tools/security/semgrep/default.nix @@ -1,6 +1,5 @@ { lib , fetchFromGitHub -, callPackage , semgrep-core , buildPythonApplication , pythonPackages @@ -11,12 +10,20 @@ }: let - common = callPackage ./common.nix { }; + common = import ./common.nix { inherit lib; }; in buildPythonApplication rec { pname = "semgrep"; - inherit (common) src version; + inherit (common) version; + src = fetchFromGitHub { + owner = "returntocorp"; + repo = "semgrep"; + rev = "v${version}"; + hash = common.srcHash; + }; + # prepare a subset of the submodules as we only need a handful + # and there are many many submodules total postPatch = (lib.concatStringsSep "\n" (lib.mapAttrsToList ( path: submodule: '' @@ -27,7 +34,7 @@ buildPythonApplication rec { ln -s ${submodule}/ ${path} '' ) - common.submodules)) + '' + passthru.submodulesSubset)) + '' cd cli ''; @@ -97,10 +104,12 @@ buildPythonApplication rec { passthru = { inherit common; + submodulesSubset = lib.mapAttrs (k: args: fetchFromGitHub args) common.submodules; updateScript = ./update.sh; }; meta = common.meta // { description = common.meta.description + " - cli"; + inherit (semgrep-core.meta) platforms; }; } diff --git a/pkgs/tools/security/semgrep/semgrep-core.nix b/pkgs/tools/security/semgrep/semgrep-core.nix index e5ce941298a4..c4846c6d91d5 100644 --- a/pkgs/tools/security/semgrep/semgrep-core.nix +++ b/pkgs/tools/security/semgrep/semgrep-core.nix @@ -1,21 +1,52 @@ -{ lib, stdenvNoCC, callPackage }: +{ lib, stdenvNoCC, fetchPypi, unzip }: let - common = callPackage ./common.nix { }; + common = import ./common.nix { inherit lib; }; in stdenvNoCC.mkDerivation rec { pname = "semgrep-core"; inherit (common) version; - inherit (common.core) src; + # fetch pre-built semgrep-core since the ocaml build is complex and relies on + # the opam package manager at some point + # pulling it out of the python wheel as r2c no longer release a built binary + # on github releases + src = + let + inherit (stdenvNoCC.hostPlatform) system; + data = common.core.${system} or (throw "Unsupported system: ${system}"); + in + fetchPypi rec { + pname = "semgrep"; + inherit version; + format = "wheel"; + dist = python; + python = "cp37.cp38.cp39.py37.py38.py39"; + inherit (data) platform hash; + }; + + nativeBuildInputs = [ unzip ]; + + # _tryUnzip from unzip's setup-hook doesn't recognise .whl + # "do not know how to unpack source archive" + # perform unpack by hand + unpackPhase = '' + runHook preUnpack + LANG=en_US.UTF-8 unzip -qq "$src" + runHook postUnpack + ''; + + dontConfigure = true; + dontBuild = true; installPhase = '' runHook preInstall - install -Dm 755 -t $out/bin semgrep-core + install -Dm 755 -t $out/bin semgrep-${version}.data/purelib/semgrep/bin/semgrep-core runHook postInstall ''; meta = common.meta // { description = common.meta.description + " - core binary"; sourceProvenance = with lib.sourceTypes; [ binaryNativeCode ]; + platforms = lib.attrNames common.core; }; } diff --git a/pkgs/tools/security/semgrep/update.sh b/pkgs/tools/security/semgrep/update.sh index 25b18edd6e2f..090d607a6928 100755 --- a/pkgs/tools/security/semgrep/update.sh +++ b/pkgs/tools/security/semgrep/update.sh @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p curl gnused jq +#!nix-shell -i bash -p curl gnused jq nix-prefetch set -euxo pipefail @@ -33,7 +33,7 @@ NEW_VERSION=$( ) # trim v prefix NEW_VERSION="${NEW_VERSION:1}" -OLD_VERSION="$(instantiateClean semgrep.common.version)" +OLD_VERSION="$(instantiateClean semgrep.passthru.common.version)" if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then echo "Already up to date" @@ -50,43 +50,54 @@ fetchgithub() { set -eo pipefail } -fetchzip() { - set +eo pipefail - nix-build -E "with import $NIXPKGS_ROOT {}; fetchzip {url = \"$1\"; sha256 = lib.fakeSha256; }" 2>&1 >/dev/null | grep "got:" | cut -d':' -f2 | sed 's| ||g' - set -eo pipefail +fetch_arch() { + VERSION=$1 + PLATFORM=$2 + nix-prefetch "{ fetchPypi }: +fetchPypi rec { + pname = \"semgrep\"; + version = \"$VERSION\"; + format = \"wheel\"; + dist = python; + python = \"cp37.cp38.cp39.py37.py38.py39\"; + platform = \"$PLATFORM\"; +} +" } replace "$OLD_VERSION" "$NEW_VERSION" "$COMMON_FILE" echo "Updating src" -OLD_HASH="$(instantiateClean semgrep.common.src.outputHash)" +OLD_HASH="$(instantiateClean semgrep.passthru.common.srcHash)" echo "Old hash $OLD_HASH" TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE" -NEW_HASH="$(fetchgithub semgrep.common.src)" +NEW_HASH="$(fetchgithub semgrep.src)" echo "New hash $NEW_HASH" replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE" echo "Updated src" -# loop through platforms for core -nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.common.core.data" --eval --strict --json \ -| jq '.[]' -r \ -| while read -r PLATFORM; do - echo "Updating core for $PLATFORM" - SUFFIX=$(instantiateClean semgrep.common.core.data."$PLATFORM".suffix) - OLD_HASH=$(instantiateClean semgrep.common.core.data."$PLATFORM".sha256) - echo "Old hash $OLD_HASH" - NEW_URL="https://github.com/returntocorp/semgrep/releases/download/v$NEW_VERSION/semgrep-v$NEW_VERSION$SUFFIX" - NEW_HASH="$(fetchzip "$NEW_URL")" - echo "New hash $NEW_HASH" +update_core_platform() { + SYSTEM=$1 + echo "Updating core src $SYSTEM" + + PLATFORM="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.platform")" + OLD_HASH="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.hash")" + echo "Old core hash $OLD_HASH" + NEW_HASH="$(fetch_arch "$NEW_VERSION" "$PLATFORM")" + echo "New core hash $NEW_HASH" replace "$OLD_HASH" "$NEW_HASH" "$COMMON_FILE" - echo "Updated core for $PLATFORM" -done + echo "Updated core src $SYSTEM" +} + +update_core_platform "x86_64-linux" +update_core_platform "x86_64-darwin" +update_core_platform "aarch64-darwin" OLD_PWD=$PWD TMPDIR="$(mktemp -d)" @@ -109,7 +120,7 @@ nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.pas echo "Updating $SUBMODULE" OLD_REV=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".rev) echo "Old commit $OLD_REV" - OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".outputHash) + OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".hash) echo "Old hash $OLD_HASH" NEW_REV=$(get_submodule_commit "$SUBMODULE") @@ -120,13 +131,12 @@ nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.pas continue fi - NEW_URL=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".url | sed "s@$OLD_REV@$NEW_REV@g") - NEW_HASH=$(nix --experimental-features nix-command hash to-sri "sha256:$(nix-prefetch-url "$NEW_URL")") + NEW_URL=$(instantiateClean semgrep.passthru.submodulesSubset."$SUBMODULE".url | sed "s@$OLD_REV@$NEW_REV@g") TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" replace "$OLD_REV" "$NEW_REV" "$COMMON_FILE" replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE" - NEW_HASH="$(fetchgithub semgrep.passthru.common.submodules."$SUBMODULE")" + NEW_HASH="$(fetchgithub semgrep.passthru.submodulesSubset."$SUBMODULE")" echo "New hash $NEW_HASH" replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE" |