2019-07-23 15:33:05 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								{ stdenv, buildPythonPackage, fetchPypi, glibcLocales, mock, pytest, botocore,
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-15 22:01:38 +00:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  testfixtures, pillow, six, twisted, w3lib, lxml, queuelib, pyopenssl,
							 | 
						
					
						
							
								
									
										
										
										
											2019-02-12 22:47:25 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  service-identity, parsel, pydispatcher, cssselect, lib }:
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-15 22:01:38 +00:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								buildPythonPackage rec {
							 | 
						
					
						
							
								
									
										
										
										
											2019-08-02 02:18:24 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  version = "1.7.3";
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-23 15:27:58 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  pname = "Scrapy";
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-15 22:01:38 +00:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-23 15:27:58 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  checkInputs = [ glibcLocales mock pytest botocore testfixtures pillow ];
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  propagatedBuildInputs = [
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    six twisted w3lib lxml cssselect queuelib pyopenssl service-identity parsel pydispatcher
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  ];
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-15 22:01:38 +00:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-12-22 22:21:54 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  patches = [
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    # Scrapy is usually installed via pip where copying all
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    # permissions makes sense. In Nix the files copied are owned by
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    # root and readonly. As a consequence scrapy can't edit the
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    # project templates.
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    ./permissions-fix.patch
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  ];
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-15 22:01:38 +00:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-23 15:27:58 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  LC_ALL="en_US.UTF-8";
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-15 22:01:38 +00:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2019-02-12 22:47:25 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  # Disable doctest plugin—enabled in the shipped pytest.ini—because it causes pytest to hang
							 | 
						
					
						
							
								
									
										
										
										
											2018-12-22 22:21:54 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  # Ignore proxy tests because requires mitmproxy
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  # Ignore test_retry_dns_error because tries to resolve an invalid dns and weirdly fails with "Reactor was unclean"
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  # Ignore xml encoding test on darwin because lxml can't find encodings https://bugs.launchpad.net/lxml/+bug/707396
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-23 15:27:58 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  checkPhase = ''
							 | 
						
					
						
							
								
									
										
										
										
											2019-07-18 19:57:42 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    substituteInPlace pytest.ini --replace "addopts = --doctest-modules" "addopts ="
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    pytest --ignore=tests/test_linkextractors_deprecated.py --ignore=tests/test_proxy_connect.py --deselect tests/test_crawl.py::CrawlTestCase::test_retry_dns_error ${lib.optionalString stdenv.isDarwin "--deselect tests/test_utils_iterators.py::LxmlXmliterTestCase::test_xmliter_encoding"}
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-23 15:27:58 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  '';
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-15 22:01:38 +00:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-23 15:27:58 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  src = fetchPypi {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    inherit pname version;
							 | 
						
					
						
							
								
									
										
										
										
											2019-08-02 02:18:24 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    sha256 = "fe5a40177960e97d42d1c752a73edb40f76a85a24076dec8535cffa499eb08c8";
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-23 15:27:58 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  };
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-15 22:01:38 +00:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-12-22 22:21:54 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  postInstall = ''
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    install -m 644 -D extras/scrapy.1 $out/share/man/man1/scrapy.1
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    install -m 644 -D extras/scrapy_bash_completion $out/share/bash-completion/completions/scrapy
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    install -m 644 -D extras/scrapy_zsh_completion $out/share/zsh/site-functions/_scrapy
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  '';
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-23 15:27:58 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								  meta = with lib; {
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								    description = "A fast high-level web crawling and web scraping framework, used to crawl websites and extract structured data from their pages";
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-29 21:06:39 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    homepage = https://scrapy.org/;
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-23 15:27:58 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    license = licenses.bsd3;
							 | 
						
					
						
							
								
									
										
										
										
											2018-12-22 22:21:54 -05:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    maintainers = with maintainers; [ drewkett marsam ];
							 | 
						
					
						
							
								
									
										
										
										
											2018-06-23 15:27:58 +02:00
										 
									 
								 
							 | 
							
								
									
										
									
								
							 | 
							
								
							 | 
							
							
								    platforms = platforms.unix;
							 | 
						
					
						
							| 
								
							 | 
							
								
							 | 
							
								
							 | 
							
							
								  };
							 | 
						
					
						
							
								
									
										
										
										
											2017-02-15 22:01:38 +00:00
										 
									 
								 
							 | 
							
								
							 | 
							
								
							 | 
							
							
								}
							 |