From 17e9f21b9c48ef488f305f3c5161743c8605b092 Mon Sep 17 00:00:00 2001 From: Tom Hunger Date: Thu, 9 Feb 2017 16:56:23 +0000 Subject: [PATCH 1/4] parsel: 1.0.3 -> 1.1.0 --- pkgs/top-level/python-packages.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 6567158c185..8ad0daa0a07 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -18146,11 +18146,11 @@ in { parsel = buildPythonPackage rec { name = "parsel-${version}"; - version = "1.0.3"; + version = "1.1.0"; src = pkgs.fetchurl { url = "mirror://pypi/p/parsel/${name}.tar.gz"; - sha256 = "9c12c370feda864c2f541cecce9bfb3a2a682c6c59c097a852e7b040dc6b8431"; + sha256 = "0a34d1c0bj1fzb5dk5744m2ag6v3b8glk4xp0amqxdan9ldbcd97"; }; buildInputs = with self; [ pytest pytestrunner ]; From dde5350971953d7bc06f1b3bb540f1806ad06f00 Mon Sep 17 00:00:00 2001 From: Tom Hunger Date: Thu, 9 Feb 2017 16:56:42 +0000 Subject: [PATCH 2/4] w3lib: 1.14.2 -> 1.17.0 --- pkgs/top-level/python-packages.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 8ad0daa0a07..d40fa082107 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -30937,13 +30937,13 @@ EOF w3lib = buildPythonPackage rec { name = "w3lib-${version}"; - version = "1.14.2"; + version = "1.17.0"; buildInputs = with self ; [ six pytest ]; src = pkgs.fetchurl { url = "mirror://pypi/w/w3lib/${name}.tar.gz"; - sha256 = "bd87eae62d208eef70869951abf05e96a8ee559714074a485168de4c5b190004"; + sha256 = "0vshh300ay5wn5hwl9qcb32m71pz5s6miy0if56vm4nggy159inq"; }; meta = { From c10b0e7bc4b001da7d0530765ba84c79f9432530 Mon Sep 17 00:00:00 2001 From: Tom Hunger Date: Thu, 9 Feb 2017 16:56:52 +0000 Subject: [PATCH 3/4] scrapy: 1.1.2 -> 1.3.1 --- pkgs/top-level/python-packages.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index d40fa082107..12d59d9847c 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -31004,7 +31004,7 @@ EOF scrapy = buildPythonPackage rec { name = "Scrapy-${version}"; - version = "1.1.2"; + version = "1.3.1"; buildInputs = with self; [ pkgs.glibcLocales mock pytest botocore testfixtures pillow ]; propagatedBuildInputs = with self; [ @@ -31020,7 +31020,7 @@ EOF src = pkgs.fetchurl { url = "mirror://pypi/S/Scrapy/${name}.tar.gz"; - sha256 = "a0a8c7bccbd598d2731ec9f267b8efbd8fb99767f826f8f2924a5610707a03d4"; + sha256 = "0s5qkxwfq842maxjd2j82ldp4dyb70kla3z5rr56z0p7ig53cbvk"; }; meta = { From 69363e96113598d95f9083346d5600cc64e349fd Mon Sep 17 00:00:00 2001 From: Tom Hunger Date: Wed, 15 Feb 2017 22:01:38 +0000 Subject: [PATCH 4/4] Move scrapy to its own module and add patch to fix broken permission code. Scrapy is usually installed via pip where copying all permissions makes sense. In Nix the files copied are owned by root and readonly. As a consequence scrapy can't edit the project templates so scrapy startproject fails. --- .../python-modules/scrapy/default.nix | 38 +++++++++++++++++++ .../scrapy/permissions-fix.patch | 28 ++++++++++++++ pkgs/top-level/python-packages.nix | 29 +------------- 3 files changed, 67 insertions(+), 28 deletions(-) create mode 100644 pkgs/development/python-modules/scrapy/default.nix create mode 100644 pkgs/development/python-modules/scrapy/permissions-fix.patch diff --git a/pkgs/development/python-modules/scrapy/default.nix b/pkgs/development/python-modules/scrapy/default.nix new file mode 100644 index 00000000000..8f3b2ef74b2 --- /dev/null +++ b/pkgs/development/python-modules/scrapy/default.nix @@ -0,0 +1,38 @@ +{ buildPythonPackage, fetchurl, glibcLocales, mock, pytest, botocore, + testfixtures, pillow, six, twisted, w3lib, lxml, queuelib, pyopenssl, + service-identity, parsel, pydispatcher, cssselect, lib }: +buildPythonPackage rec { + name = "Scrapy-${version}"; + version = "1.3.1"; + + buildInputs = [ glibcLocales mock pytest botocore testfixtures pillow ]; + propagatedBuildInputs = [ + six twisted w3lib lxml cssselect queuelib pyopenssl service-identity parsel pydispatcher + ]; + + # Scrapy is usually installed via pip where copying all + # permissions makes sense. In Nix the files copied are owned by + # root and readonly. As a consequence scrapy can't edit the + # project templates. + patches = [ ./permissions-fix.patch ]; + + LC_ALL="en_US.UTF-8"; + + checkPhase = '' + py.test --ignore=tests/test_linkextractors_deprecated.py --ignore=tests/test_proxy_connect.py + # The ignored tests require mitmproxy, which depends on protobuf, but it's disabled on Python3 + ''; + + src = fetchurl { + url = "mirror://pypi/S/Scrapy/${name}.tar.gz"; + sha256 = "0s5qkxwfq842maxjd2j82ldp4dyb70kla3z5rr56z0p7ig53cbvk"; + }; + + meta = with lib; { + description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages"; + homepage = "http://scrapy.org/"; + license = licenses.bsd3; + maintainers = with maintainers; [ drewkett ]; + platforms = platforms.linux; + }; +} diff --git a/pkgs/development/python-modules/scrapy/permissions-fix.patch b/pkgs/development/python-modules/scrapy/permissions-fix.patch new file mode 100644 index 00000000000..5ea5269c799 --- /dev/null +++ b/pkgs/development/python-modules/scrapy/permissions-fix.patch @@ -0,0 +1,28 @@ +diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py +index 5941066..89f8edb 100644 +--- a/scrapy/commands/startproject.py ++++ b/scrapy/commands/startproject.py +@@ -4,7 +4,7 @@ import os + import string + from importlib import import_module + from os.path import join, exists, abspath +-from shutil import ignore_patterns, move, copy2, copystat ++from shutil import ignore_patterns, move, copyfile, copystat + + import scrapy + from scrapy.commands import ScrapyCommand +@@ -76,8 +76,7 @@ class Command(ScrapyCommand): + if os.path.isdir(srcname): + self._copytree(srcname, dstname) + else: +- copy2(srcname, dstname) +- copystat(src, dst) ++ copyfile(srcname, dstname) + + def run(self, args, opts): + if len(args) not in (1, 2): +@@ -118,4 +117,3 @@ class Command(ScrapyCommand): + _templates_base_dir = self.settings['TEMPLATES_DIR'] or \ + join(scrapy.__path__[0], 'templates') + return join(_templates_base_dir, 'project') +- diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 12d59d9847c..03f6d7ce07e 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -31002,35 +31002,8 @@ EOF }; }; - scrapy = buildPythonPackage rec { - name = "Scrapy-${version}"; - version = "1.3.1"; + scrapy = callPackage ../development/python-modules/scrapy { }; - buildInputs = with self; [ pkgs.glibcLocales mock pytest botocore testfixtures pillow ]; - propagatedBuildInputs = with self; [ - six twisted w3lib lxml cssselect queuelib pyopenssl service-identity parsel pydispatcher - ]; - - LC_ALL="en_US.UTF-8"; - - checkPhase = '' - py.test --ignore=tests/test_linkextractors_deprecated.py --ignore=tests/test_proxy_connect.py - # The ignored tests require mitmproxy, which depends on protobuf, but it's disabled on Python3 - ''; - - src = pkgs.fetchurl { - url = "mirror://pypi/S/Scrapy/${name}.tar.gz"; - sha256 = "0s5qkxwfq842maxjd2j82ldp4dyb70kla3z5rr56z0p7ig53cbvk"; - }; - - meta = { - description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages"; - homepage = "http://scrapy.org/"; - license = licenses.bsd3; - maintainers = with maintainers; [ drewkett ]; - platforms = platforms.linux; - }; - }; pandocfilters = buildPythonPackage rec{ version = "1.4.1"; pname = "pandocfilters";