1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
|
{ lib
, stdenv
, buildPythonPackage
, braceexpand
, imageio
, lmdb
, msgpack
, numpy
, pytestCheckHook
, pyyaml
, setuptools
, torch
, torchvision
, wheel
, fetchFromGitHub
}:
buildPythonPackage rec {
pname = "webdataset";
version = "0.2.86";
pyproject = true;
src = fetchFromGitHub {
owner = "webdataset";
repo = "webdataset";
rev = "refs/tags/${version}";
hash = "sha256-aTjxoSoQ9LH4gcFmV+7Aj0HNIpvsFHTrxFUpAtB3nkM=";
};
nativeBuildInputs = [
setuptools
wheel
];
propagatedBuildInputs = [
braceexpand
numpy
pyyaml
];
nativeCheckInputs = [
pytestCheckHook
imageio
torch
torchvision
msgpack
lmdb
];
pythonImportsCheck = [ "webdataset" ];
disabledTests = [
# requires network
"test_batched"
"test_dataloader"
"test_decode_handlers"
"test_decoder"
"test_decoder"
"test_download"
"test_handlers"
"test_pipe"
"test_shard_syntax"
"test_torchvision"
"test_torchvision"
"test_unbatched"
"test_yaml3"
] ++ lib.optionals stdenv.isDarwin [
# pickling error
"test_background_download"
] ++ lib.optionals (stdenv.isAarch64 && stdenv.isLinux) [
# segfaults on aarch64-linux
"test_webloader"
"test_webloader2"
"test_webloader_repeat"
"test_webloader_unbatched"
];
meta = with lib; {
description = "A high-performance Python-based I/O system for large (and small) deep learning problems, with strong support for PyTorch";
homepage = "https://github.com/webdataset/webdataset";
license = licenses.bsd3;
maintainers = with maintainers; [ iynaix ];
};
}
|