diff --git a/pkgs/tools/backup/grab-site/default.nix b/pkgs/tools/backup/grab-site/default.nix new file mode 100644 index 00000000000..7209afd8e7e --- /dev/null +++ b/pkgs/tools/backup/grab-site/default.nix @@ -0,0 +1,30 @@ +{ stdenv, python3Packages, fetchFromGitHub }: + +python3Packages.buildPythonApplication rec { + version = "2.1.11"; + name = "grab-site-${version}"; + + src = fetchFromGitHub { + rev = "${version}"; + owner = "ludios"; + repo = "grab-site"; + sha256 = "0w24ngr2b7nipqiwkxpql2467b5aq2vbknkb0sry6a457kb5ppsl"; + }; + + propagatedBuildInputs = with python3Packages; [ + click ludios_wpull manhole lmdb autobahn fb-re2 websockets cchardet + ]; + + checkPhase = '' + export PATH=$PATH:$out/bin + bash ./tests/offline-tests + ''; + + meta = with stdenv.lib; { + description = "Crawler for web archiving with WARC output"; + homepage = https://github.com/ludios/grab-site; + license = licenses.mit; + maintainers = with maintainers; [ ivan ]; + platforms = platforms.all; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 209a90a1601..3e392af3f65 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -10004,6 +10004,8 @@ in gperftools = callPackage ../development/libraries/gperftools { }; + grab-site = callPackage ../tools/backup/grab-site { }; + grib-api = callPackage ../development/libraries/grib-api { }; grpc = callPackage ../development/libraries/grpc { };