diff options
Diffstat (limited to 'nixpkgs/pkgs/development/python-modules/apache-airflow/default.nix')
| -rw-r--r-- | nixpkgs/pkgs/development/python-modules/apache-airflow/default.nix | 218 |
1 files changed, 129 insertions, 89 deletions
diff --git a/nixpkgs/pkgs/development/python-modules/apache-airflow/default.nix b/nixpkgs/pkgs/development/python-modules/apache-airflow/default.nix index 82d7a8cc32af..dea5776fb616 100644 --- a/nixpkgs/pkgs/development/python-modules/apache-airflow/default.nix +++ b/nixpkgs/pkgs/development/python-modules/apache-airflow/default.nix @@ -1,165 +1,202 @@ { lib , stdenv +, python , buildPythonPackage , fetchFromGitHub -, fetchpatch +, writeText , alembic +, argcomplete +, attrs +, blinker , cached-property -, configparser +, cattrs +, clickclick , colorlog , croniter +, cryptography , dill , flask , flask-appbuilder -, flask-admin , flask-caching , flask_login -, flask-swagger , flask_wtf -, flask-bcrypt -, funcsigs -, future , GitPython +, graphviz , gunicorn +, httpx , iso8601 -, json-merge-patch +, importlib-resources +, importlib-metadata +, inflection +, itsdangerous , jinja2 -, ldap3 -, lxml +, jsonschema , lazy-object-proxy +, lockfile , markdown +, markupsafe +, marshmallow-oneofschema +, numpy +, openapi-spec-validator , pandas , pendulum , psutil , pygments +, pyjwt , python-daemon , python-dateutil -, requests +, python-nvd3 +, python-slugify +, python3-openid +, pyyaml +, rich , setproctitle -, snakebite , sqlalchemy +, sqlalchemy-jsonfield +, swagger-ui-bundle , tabulate , tenacity , termcolor -, text-unidecode -, thrift -, tzlocal , unicodecsv -, zope_deprecation -, nose -, pythonOlder -, pythonAtLeast +, werkzeug +, pytest +, freezegun +, mkYarnPackage }: +let -buildPythonPackage rec { - pname = "apache-airflow"; - version = "1.10.5"; - # Upstream does not yet support python 3.8 - # https://github.com/apache/airflow/issues/8674 - disabled = pythonOlder "3.5" || pythonAtLeast "3.8"; + version = "2.1.2"; - src = fetchFromGitHub rec { + airflow-src = fetchFromGitHub rec { owner = "apache"; repo = "airflow"; rev = version; - sha256 = "14fmhfwx977c9jdb2kgm93i6acx43l45ggj30rb37r68pzpb6l6h"; + sha256 = "sha256-Q0l2c1tuxcoE65zgdxnv/j1TIoQzaNoEFCYHvqN+Bzk="; }; - patches = [ - # Not yet accepted: https://github.com/apache/airflow/pull/6562 - (fetchpatch { - name = "avoid-warning-from-abc.collections"; - url = "https://patch-diff.githubusercontent.com/raw/apache/airflow/pull/6562.patch"; - sha256 = "0swpay1qlb7f9kgc56631s1qd9k82w4nw2ggvkm7jvxwf056k61z"; - }) - # Not yet accepted: https://github.com/apache/airflow/pull/6561 - (fetchpatch { - name = "pendulum2-compatibility"; - url = "https://patch-diff.githubusercontent.com/raw/apache/airflow/pull/6561.patch"; - sha256 = "17hw8qyd4zxvib9zwpbn32p99vmrdz294r31gnsbkkcl2y6h9knk"; - }) - ]; + # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree. + # This replicates this shell script, fixing bugs in yarn.lock and package.json + + airflow-frontend = mkYarnPackage { + name = "airflow-frontend"; + + src = "${airflow-src}/airflow/www"; + packageJSON = ./package.json; + yarnLock = ./yarn.lock; + yarnNix = ./yarn.nix; + + distPhase = "true"; + + configurePhase = '' + cp -r $node_modules node_modules + ''; + + buildPhase = '' + yarn --offline build + find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5 + ''; + + installPhase = '' + mkdir -p $out/static/ + cp -r static/dist $out/static + ''; + }; + +in +buildPythonPackage rec { + pname = "apache-airflow"; + inherit version; + src = airflow-src; propagatedBuildInputs = [ alembic + argcomplete + attrs + blinker cached-property + cattrs + clickclick colorlog - configparser croniter + cryptography dill flask - flask-admin flask-appbuilder - flask-bcrypt flask-caching flask_login - flask-swagger flask_wtf - funcsigs - future GitPython + graphviz gunicorn + httpx iso8601 - json-merge-patch + importlib-resources + importlib-metadata + inflection + itsdangerous jinja2 - ldap3 - lxml + jsonschema lazy-object-proxy + lockfile markdown + markupsafe + marshmallow-oneofschema + numpy + openapi-spec-validator pandas pendulum psutil pygments + pyjwt python-daemon python-dateutil - requests + python-nvd3 + python-slugify + python3-openid + pyyaml + rich setproctitle sqlalchemy + sqlalchemy-jsonfield + swagger-ui-bundle tabulate tenacity termcolor - text-unidecode - thrift - tzlocal unicodecsv - zope_deprecation + werkzeug + ]; + + buildInputs = [ + airflow-frontend ]; checkInputs = [ - snakebite - nose + freezegun + pytest ]; + INSTALL_PROVIDERS_FROM_SOURCES = "true"; + postPatch = '' - substituteInPlace setup.py \ + substituteInPlace setup.cfg \ + --replace "importlib_resources~=1.4" "importlib_resources" \ + --replace "importlib_metadata~=1.7" "importlib_metadata" \ + --replace "tenacity~=6.2.0" "tenacity" \ + --replace "pyjwt<2" "pyjwt" \ --replace "flask>=1.1.0, <2.0" "flask" \ - --replace "jinja2>=2.10.1, <2.11.0" "jinja2" \ - --replace "pandas>=0.17.1, <1.0.0" "pandas" \ - --replace "flask-caching>=1.3.3, <1.4.0" "flask-caching" \ - --replace "flask-appbuilder>=1.12.5, <2.0.0" "flask-appbuilder" \ - --replace "flask-admin==1.5.3" "flask-admin" \ --replace "flask-login>=0.3, <0.5" "flask-login" \ - --replace "cached_property~=1.5" "cached_property" \ - --replace "dill>=0.2.2, <0.3" "dill" \ - --replace "configparser>=3.5.0, <3.6.0" "configparser" \ - --replace "colorlog==4.0.2" "colorlog" \ - --replace "funcsigs==1.0.0" "funcsigs" \ - --replace "flask-swagger==0.2.13" "flask-swagger" \ - --replace "python-daemon>=2.1.1, <2.2" "python-daemon" \ - --replace "alembic>=1.0, <2.0" "alembic" \ - --replace "markdown>=2.5.2, <3.0" "markdown" \ - --replace "future>=0.16.0, <0.17" "future" \ - --replace "tenacity==4.12.0" "tenacity" \ - --replace "text-unidecode==1.2" "text-unidecode" \ - --replace "tzlocal>=1.4,<2.0.0" "tzlocal" \ - --replace "sqlalchemy~=1.3" "sqlalchemy" \ - --replace "gunicorn>=19.5.0, <20.0" "gunicorn" - - # dumb-init is only needed for CI and Docker, not relevant for NixOS. - substituteInPlace setup.py \ - --replace "'dumb-init>=1.2.2'," "" - - substituteInPlace tests/core.py \ + --replace "flask-wtf>=0.14.3, <0.15" "flask-wtf" \ + --replace "jinja2>=2.10.1, <2.12.0" "jinja2" \ + --replace "attrs>=20.0, <21.0" "attrs" \ + --replace "cattrs~=1.1, <1.7.0" "cattrs" \ + --replace "markupsafe>=1.1.1, <2.0" "markupsafe" \ + --replace "docutils<0.17" "docutils" \ + --replace "sqlalchemy>=1.3.18, <1.4" "sqlalchemy" \ + --replace "sqlalchemy_jsonfield~=1.0" "sqlalchemy-jsonfield" \ + --replace "werkzeug~=1.0, >=1.0.1" "werkzeug" \ + --replace "itsdangerous>=1.1.0, <2.0" "itsdangerous" + + substituteInPlace tests/core/test_core.py \ --replace "/bin/bash" "${stdenv.shell}" ''; @@ -174,11 +211,14 @@ buildPythonPackage rec { export PATH=$PATH:$out/bin airflow version - airflow initdb - airflow resetdb -y - nosetests tests.core.CoreTest - ## all tests - # nosetests --cover-package=airflow + airflow db init + airflow db reset -y + + pytest tests/core/test_core.py + ''; + + postInstall = '' + cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static ''; meta = with lib; { |