diff options
17 files changed, 312 insertions, 0 deletions
diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix new file mode 100644 index 000000000000..989107a2fbef --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/default.nix @@ -0,0 +1,106 @@ +{ lib, stdenv +, fetchgit, fetchFromGitHub, fetchurl +, writeShellScript, runCommand +, rustPlatform, jq, nix-prefetch-git, xe, curl +}: + +# TODO: move to carnix or https://github.com/kolloch/crate2nix +let + # to update: + # 1) change all these hashes + # 2) nix-build -A tree-sitter.updater.update-all-grammars + # 3) run the script that is output by that (it updates ./grammars) + version = "0.15.7"; + sha256 = "0q6w8wl4a4s49xlgbv531pandzrj3n12hc1cwfshzcgikx303dg0"; + sha256Js = "11ig4cc2m85siyhafh4hq9sjb5if4gfwsf9k87izkxpiyflda0wp"; + sha256Wasm = "1zm4bvjri8ivhah3sy22mx6jbvibgbn2hk67d148j3nyka3y4gc0"; + cargoSha256 = "0ls9cb2p6cgqvnrmx72n79ga7687n8mzhh7n8n1pzv11r6cah9ki"; + + + src = fetchFromGitHub { + owner = "tree-sitter"; + repo = "tree-sitter"; + rev = version; + inherit sha256; + fetchSubmodules = true; + }; + + fetchDist = {file, sha256}: fetchurl { + url = "https://github.com/tree-sitter/tree-sitter/releases/download/${version}/${file}"; + inherit sha256; + }; + + # TODO: not distributed anymore; needed for the web-ui module, + # see also the disable-web-ui patch. + # TODO: build those instead of downloading prebuilt + # js = fetchDist { + # file = "tree-sitter.js"; + # sha256 = sha256Js; + # }; + # wasm = fetchDist { + # file = "tree-sitter.wasm"; + # sha256 = sha256Wasm; + # }; + + update-all-grammars = import ./update.nix { + inherit writeShellScript nix-prefetch-git curl jq xe src; + }; + + grammars = + let fetch = + (v: fetchgit {inherit (v) url rev sha256 fetchSubmodules; }); + in runCommand "grammars" {} ('' + mkdir $out + '' + (lib.concatStrings (lib.mapAttrsToList + (name: grammar: "ln -s ${fetch grammar} $out/${name}\n") + (import ./grammars)))); + + +in rustPlatform.buildRustPackage { + pname = "tree-sitter"; + inherit version; + inherit src; + + patches = [ + # the web ui requires tree-sitter compiled to js and wasm + ./disable-web-ui.patch + ]; + + postPatch = '' + # needed for the tests + rm -rf test/fixtures/grammars + ln -s ${grammars} test/fixtures/grammars + ''; + + passthru = { + updater = { + inherit update-all-grammars; + }; + inherit grammars; + }; + + inherit cargoSha256; + + meta = { + homepage = "https://github.com/tree-sitter/tree-sitter"; + description = "A parser generator tool and an incremental parsing library"; + longDescription = '' + Tree-sitter is a parser generator tool and an incremental parsing library. + It can build a concrete syntax tree for a source file and efficiently update the syntax tree as the source file is edited. + + Tree-sitter aims to be: + + * General enough to parse any programming language + * Fast enough to parse on every keystroke in a text editor + * Robust enough to provide useful results even in the presence of syntax errors + * Dependency-free so that the runtime library (which is written in pure C) can be embedded in any application + ''; + platforms = lib.platforms.all; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ Profpatsch ]; + # Darwin needs some more work with default libraries + # Aarch has test failures with how tree-sitter compiles the generated C files + broken = stdenv.isDarwin || stdenv.isAarch64; + }; + +} diff --git a/pkgs/development/tools/parsing/tree-sitter/disable-web-ui.patch b/pkgs/development/tools/parsing/tree-sitter/disable-web-ui.patch new file mode 100644 index 000000000000..6065aa9e7fdb --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/disable-web-ui.patch @@ -0,0 +1,37 @@ +diff --git a/cli/src/lib.rs b/cli/src/lib.rs +index 33a9904f..633032d7 100644 +--- a/cli/src/lib.rs ++++ b/cli/src/lib.rs +@@ -8,7 +8,7 @@ pub mod parse; + pub mod test; + pub mod util; + pub mod wasm; +-pub mod web_ui; ++// pub mod web_ui; + + #[cfg(test)] + mod tests; +diff --git a/cli/src/main.rs b/cli/src/main.rs +index 23e7fc1a..9d784c8a 100644 +--- a/cli/src/main.rs ++++ b/cli/src/main.rs +@@ -4,7 +4,7 @@ use std::{env, fs, u64}; + use std::path::Path; + use std::process::exit; + use tree_sitter_cli::{ +- config, error, generate, highlight, loader, logger, parse, test, wasm, web_ui, ++ config, error, generate, highlight, loader, logger, parse, test, wasm, + }; + + const BUILD_VERSION: &'static str = env!("CARGO_PKG_VERSION"); +@@ -250,7 +250,9 @@ fn run() -> error::Result<()> { + let grammar_path = current_dir.join(matches.value_of("path").unwrap_or("")); + wasm::compile_language_to_wasm(&grammar_path, matches.is_present("docker"))?; + } else if matches.subcommand_matches("web-ui").is_some() { +- web_ui::serve(¤t_dir); ++ print!("ERROR: web-ui is not available in the nixpkgs tree-sitter-cli at the moment."); ++ std::process::exit(1); ++ // web_ui::serve(¤t_dir); + } + + Ok(()) diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix new file mode 100644 index 000000000000..b6451bce7282 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix @@ -0,0 +1,15 @@ +{ + bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json)); + c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json)); + cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json)); + embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json)); + go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json)); + html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json)); + javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json)); + json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json)); + python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json)); + # wasn’t able to check out with fetchgit + # ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json)); + rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json)); + typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json)); +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-bash.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-bash.json new file mode 100644 index 000000000000..c367d24d09f6 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-bash.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-bash", + "rev": "7b8adcf484e27b6c1a707ce28123e55dd4b7a844", + "date": "2019-07-26T14:05:41-06:00", + "sha256": "047p51ab4fqm55xqss6z74iyj1hlndql97dv9fifckczx3d5xn5g", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c.json new file mode 100644 index 000000000000..1342f7d092b0 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-c", + "rev": "22decdc361767838dd36f1da4125b35b5b9a3c28", + "date": "2019-07-02T15:49:42-07:00", + "sha256": "03f9g49l4g2l4hlafr3xhvi8d3a491k5zz4bxpq7391l5wgjy3zi", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json new file mode 100644 index 000000000000..8c7d10f2654f --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-cpp", + "rev": "f5afa0ee48ad1dc067ed6fe1aa2cfd2a3ea5d443", + "date": "2019-08-06T17:23:46-07:00", + "sha256": "1w9zjqj232fcagqfqd8qi4kmvr655s4ivllrm27973sda4xq557h", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-embedded-template.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-embedded-template.json new file mode 100644 index 000000000000..89940ffec8cb --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-embedded-template.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-embedded-template", + "rev": "71955edec8cb762f63e94cf062fc96b52b9ae609", + "date": "2019-07-17T15:55:22-07:00", + "sha256": "1ar2n1z2h194lb3isbdkmvhn8w78j4a62nbh105w3jl1sxb4qpsa", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-go.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-go.json new file mode 100644 index 000000000000..6a354f5c2e23 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-go.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-go", + "rev": "475571bb5bdb9b229c6be3843d4c71ba747688fd", + "date": "2019-07-17T15:51:06-07:00", + "sha256": "1cg5qpifrvpnsi0iy26g156xib2qa55vlna41hw6c70kx8ibvl9z", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-html.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-html.json new file mode 100644 index 000000000000..732d2dda40b6 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-html.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-html", + "rev": "aeb2f456b8c6a60b8475d075889d476a165cde57", + "date": "2019-07-17T15:57:54-07:00", + "sha256": "0ba8zi65kja6p7f5h7pa7kxqa3mj29ysjrvl84am24vy5ik4zz3z", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json new file mode 100644 index 000000000000..a1e29e1504ea --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-javascript", + "rev": "a730b5c210904e2e3c1f601125a059fde1b35850", + "date": "2019-08-08T14:13:17-07:00", + "sha256": "1cr0vikbzrklksjj07fh34a5cabkgbpkbxwiw2alnana3zzzdhnq", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-json.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-json.json new file mode 100644 index 000000000000..2bb0dcd4d8f2 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-json.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-json", + "rev": "337f55be9b9b1ccb0baa7763bfe014a94acea7ea", + "date": "2019-05-23T11:02:26-04:00", + "sha256": "0amh4qrjj3fli9c0z6p61z9d7496sqq54i1gh2vrghgnbbyaa6mz", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json new file mode 100644 index 000000000000..f46695edfa85 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-python", + "rev": "4c22de0944cd42a5c86ade7ef7097033604796f8", + "date": "2019-07-30T15:35:03-04:00", + "sha256": "1p12h7hj1ak15fyk4gw9wcmgzydd4z5mikhjp54mn1q4vfw175p3", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json new file mode 100644 index 000000000000..e2f9928c8702 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-ruby", + "rev": "db91c934ff9d3d4ea67111a0f581532c49c3a6b3", + "date": "2019-07-26T15:51:54-06:00", + "sha256": "1ir1nqpz0c0hnsqzp90w2iw1gy3z3nqil2fm4n3zmid5di7c98dg", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json new file mode 100644 index 000000000000..328b337ed5a7 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-rust", + "rev": "3f956b18a6b0a576ed238cc69d5e3f413bd547b1", + "date": "2019-07-18T11:44:02-07:00", + "sha256": "0dwxg3pqirqm1lvl5x0q9djavfri9ffk5diygqzjnx53rwqhyzj8", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json new file mode 100644 index 000000000000..61507888154d --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json @@ -0,0 +1,7 @@ +{ + "url": "https://github.com/tree-sitter/tree-sitter-typescript", + "rev": "ab9ab6cced868ee3e096f33fa21fd9d356c92e1a", + "date": "2019-08-08T14:27:32-07:00", + "sha256": "11r0vj1dhv0my2cr442mwvaav8ljygsns20w51mwg7328vlz90q3", + "fetchSubmodules": false +} diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix new file mode 100644 index 000000000000..aa87df6bed10 --- /dev/null +++ b/pkgs/development/tools/parsing/tree-sitter/update.nix @@ -0,0 +1,68 @@ +{ writeShellScript, nix-prefetch-git +, curl, jq, xe +, src }: + +let + # print all the grammar names mentioned in the fetch-fixtures script + getGrammarNames = writeShellScript "get-grammars.sh" '' + set -euo pipefail + sed -ne 's/^fetch_grammar \(\S*\).*$/\1/p' \ + ${src}/script/fetch-fixtures + ''; + + # TODO + urlEscape = x: x; + # TODO + urlEscapeSh = writeShellScript "escape-url" ''printf '%s' "$1"''; + + # generic bash script to find the latest github release for a repo + latestGithubRelease = { owner }: writeShellScript "latest-github-release" '' + set -euo pipefail + repo="$1" + res=$(${curl}/bin/curl \ + --silent \ + "https://api.github.com/repos/${urlEscape owner}/$(${urlEscapeSh} "$repo")/releases/latest") + if [[ "$(printf "%s" "$res" | ${jq}bin/jq '.message')" =~ "rate limit" ]]; then + echo "rate limited" >&2 + fi + release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name') + # github sometimes returns an empty list even tough there are releases + if [ "$release" = "null" ]; then + echo "uh-oh, latest for $repo is not there, using HEAD" >&2 + release="HEAD" + fi + echo "$release" + ''; + + # update one tree-sitter grammar repo and print their nix-prefetch-git output + updateGrammar = { owner }: writeShellScript "update-grammar.sh" '' + set -euo pipefail + repo="$1" + latest="$(${latestGithubRelease { inherit owner; }} "$repo")" + echo "Fetching latest release ($latest) of $repo …" >&2 + ${nix-prefetch-git}/bin/nix-prefetch-git \ + --quiet \ + --no-deepClone \ + --url "https://github.com/${urlEscape owner}/$(${urlEscapeSh} "$repo")" \ + --rev "$latest" + ''; + + update-all-grammars = writeShellScript "update-all-grammars.sh" '' + set -euo pipefail + grammarNames=$(${getGrammarNames}) + outputDir="${toString ./.}/grammars" + mkdir -p "$outputDir" + updateCommand=$(printf \ + '${updateGrammar { owner = "tree-sitter"; }} "$1" > "%s/$1.json"' \ + "$outputDir") + printf '%s' "$grammarNames" \ + | ${xe}/bin/xe printf "tree-sitter-%s\n" {} \ + | ${xe}/bin/xe -j2 -s "$updateCommand" + ( echo "{" + printf '%s' "$grammarNames" \ + | ${xe}/bin/xe -s 'printf " %s = (builtins.fromJSON (builtins.readFile ./tree-sitter-%s.json));\n" "$1" "$1"' + echo "}" ) \ + > "$outputDir/default.nix" + ''; + +in update-all-grammars diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 29b3ef2a75d5..df43828460b7 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -10067,6 +10067,8 @@ in travis = callPackage ../development/tools/misc/travis { }; + tree-sitter = callPackage ../development/tools/parsing/tree-sitter { }; + trellis = callPackage ../development/tools/trellis { }; ttyd = callPackage ../servers/ttyd { }; |