From 5d7ef923cb10e02121b84f8ee98ad891d7f35630 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Fri, 10 Apr 2020 22:58:46 +0300
Subject: [PATCH 01/26] python.pkgs.sane: init at 2.8.2

---
 .../python-modules/sane/default.nix           | 27 +++++++++++++++++++
 pkgs/top-level/python-packages.nix            |  4 +++
 2 files changed, 31 insertions(+)
 create mode 100644 pkgs/development/python-modules/sane/default.nix

diff --git a/pkgs/development/python-modules/sane/default.nix b/pkgs/development/python-modules/sane/default.nix
new file mode 100644
index 00000000000..2359ff0aa7f
--- /dev/null
+++ b/pkgs/development/python-modules/sane/default.nix
@@ -0,0 +1,27 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, saneBackends
+}:
+
+buildPythonPackage rec {
+  pname = "sane";
+  version = "2.8.2";
+
+  src = fetchPypi {
+    inherit version;
+    pname = "python-sane";
+    sha256 = "0sri01h9sld6w7vgfhwp29n5w19g6idz01ba2giwnkd99k1y2iqg";
+  };
+
+  buildInputs = [
+    saneBackends
+  ];
+
+  meta = with lib; {
+    homepage = "https://github.com/python-pillow/Sane";
+    description = "Python interface to the SANE scanner and frame grabber ";
+    license = licenses.mit;
+    maintainers = with maintainers; [ doronbehar ];
+  };
+}
diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix
index 162f837fd28..82a57b0192c 100644
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@@ -6452,6 +6452,10 @@ in {
 
   salmon-mail = callPackage ../development/python-modules/salmon-mail { };
 
+  sane = callPackage ../development/python-modules/sane {
+    inherit (pkgs) saneBackends;
+  };
+
   sampledata = callPackage ../development/python-modules/sampledata { };
 
   samplerate = callPackage ../development/python-modules/samplerate { };

From cc7ce7611924425b4f269da7a9bc3ca7d3ac8514 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Sat, 5 Dec 2020 14:14:20 +0200
Subject: [PATCH 02/26] ocrfeeder: init at 0.8.3

---
 .../graphics/ocrfeeder/default.nix            | 73 +++++++++++++++++++
 pkgs/top-level/all-packages.nix               |  2 +
 2 files changed, 75 insertions(+)
 create mode 100644 pkgs/applications/graphics/ocrfeeder/default.nix

diff --git a/pkgs/applications/graphics/ocrfeeder/default.nix b/pkgs/applications/graphics/ocrfeeder/default.nix
new file mode 100644
index 00000000000..28fe58183fa
--- /dev/null
+++ b/pkgs/applications/graphics/ocrfeeder/default.nix
@@ -0,0 +1,73 @@
+{ stdenv
+, fetchurl
+, pkg-config
+, gtk3
+, gtkspell3
+, isocodes
+, goocanvas2
+, intltool
+, itstool
+, libxml2
+, gnome3
+, python3
+, gobject-introspection
+, wrapGAppsHook
+, tesseract4
+, extraOcrEngines ? [] # other supported engines are: ocrad gocr cuneiform
+}:
+
+stdenv.mkDerivation rec {
+  pname = "ocrfeeder";
+  version = "0.8.3";
+
+  src = fetchurl {
+    url = "mirror://gnome/sources/${pname}/${stdenv.lib.versions.majorMinor version}/${pname}-${version}.tar.xz";
+    sha256 = "12f5gnq92ffnd5zaj04df7jrnsdz1zn4zcgpbf5p9qnd21i2y529";
+  };
+
+  nativeBuildInputs = [
+    pkg-config
+    wrapGAppsHook
+    intltool
+    itstool
+    libxml2
+  ];
+
+  buildInputs = [
+    gtk3
+    gobject-introspection
+    goocanvas2
+    gtkspell3
+    isocodes
+    (python3.withPackages(ps: with ps; [
+      pyenchant
+      sane
+      pillow
+      reportlab
+      odfpy
+      pygobject3
+    ]))
+  ];
+
+  # https://gitlab.gnome.org/GNOME/ocrfeeder/-/issues/22
+  postConfigure = ''
+    substituteInPlace src/ocrfeeder/util/constants.py \
+      --replace /usr/share/xml/iso-codes ${isocodes}/share/xml/iso-codes
+  '';
+
+  enginesPath = stdenv.lib.makeBinPath ([
+    tesseract4
+  ] ++ extraOcrEngines);
+
+  preFixup = ''
+    gappsWrapperArgs+=(--prefix PATH : "${enginesPath}")
+  '';
+
+  meta = with stdenv.lib; {
+    homepage = "https://wiki.gnome.org/Apps/OCRFeeder";
+    description = "Complete Optical Character Recognition and Document Analysis and Recognition program";
+    maintainers = with maintainers; [ doronbehar ];
+    license = licenses.gpl3Plus;
+    platforms = platforms.linux;
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 94e18adbc2a..65e38258df7 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -2460,6 +2460,8 @@ in
 
   ocrmypdf = callPackage ../tools/text/ocrmypdf { };
 
+  ocrfeeder = callPackage ../applications/graphics/ocrfeeder { };
+
   onboard = callPackage ../applications/misc/onboard { };
 
   oneshot = callPackage ../tools/networking/oneshot { };

From 08f1b2632e9d5ed9b1791238001c51d133858f1f Mon Sep 17 00:00:00 2001
From: William Casarin <jb55@jb55.com>
Date: Fri, 18 Dec 2020 06:29:29 -0800
Subject: [PATCH 03/26] notcurses: init at 2.1.0

This is a neat ncurses alternative. See the demo here:

https://www.youtube.com/watch?v=cYhZ7myXyyg

Here's something interesting to package after this:

https://github.com/dankamongmen/growlight

Message-Id: <20201218142929.9912-1-jb55@jb55.com>
---
 .../libraries/notcurses/default.nix           | 49 +++++++++++++++++++
 pkgs/top-level/all-packages.nix               |  2 +
 2 files changed, 51 insertions(+)
 create mode 100644 pkgs/development/libraries/notcurses/default.nix

diff --git a/pkgs/development/libraries/notcurses/default.nix b/pkgs/development/libraries/notcurses/default.nix
new file mode 100644
index 00000000000..189e36c1b44
--- /dev/null
+++ b/pkgs/development/libraries/notcurses/default.nix
@@ -0,0 +1,49 @@
+{ stdenv, cmake, pkgconfig, pandoc, libunistring, ncurses, ffmpeg,
+  fetchFromGitHub, lib,
+  multimediaSupport ? true
+}:
+let
+  version = "2.1.0";
+in
+stdenv.mkDerivation {
+  pname = "notcurses";
+  inherit version;
+
+  outputs = [ "out" "dev" ];
+
+  nativeBuildInputs = [ cmake pkgconfig pandoc ];
+
+  buildInputs = [ libunistring ncurses ]
+    ++ lib.optional multimediaSupport ffmpeg;
+
+  cmakeFlags =
+    [ "-DUSE_QRCODEGEN=OFF" ]
+    ++ lib.optional (!multimediaSupport) "-DUSE_MULTIMEDIA=none";
+
+  src = fetchFromGitHub {
+    owner  = "dankamongmen";
+    repo   = "notcurses";
+    rev    = "v${version}";
+    sha256 = "0jvngg40c1sqf85kqy6ya0vflpxsj7j4g6cw609992rifaghxiny";
+  };
+
+  meta = {
+    description = "blingful TUIs and character graphics";
+
+    longDescription = ''
+      A library facilitating complex TUIs on modern terminal emulators,
+      supporting vivid colors, multimedia, and Unicode to the maximum degree
+      possible. Things can be done with Notcurses that simply can't be done
+      with NCURSES.
+
+      It is not a source-compatible X/Open Curses implementation, nor a
+      replacement for NCURSES on existing systems.
+    '';
+
+    homepage = "https://github.com/dankamongmen/notcurses";
+
+    license = lib.licenses.asl20;
+    platforms = lib.platforms.all;
+    maintainers = with lib.maintainers; [ jb55 ];
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 2554f3584e4..14fba89bd03 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -15085,6 +15085,8 @@ in
 
   notify-sharp = callPackage ../development/libraries/notify-sharp { };
 
+  notcurses = callPackage ../development/libraries/notcurses { };
+
   ncurses5 = ncurses.override {
     abiVersion = "5";
   };

From 711c65208811a6ab371cb2be2718fd6b8441a0ed Mon Sep 17 00:00:00 2001
From: Lucas Hoffmann <lucc@posteo.de>
Date: Sat, 19 Dec 2020 11:01:05 +0100
Subject: [PATCH 04/26] vicious: 2.3.1 -> 2.5.0

---
 pkgs/top-level/lua-packages.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkgs/top-level/lua-packages.nix b/pkgs/top-level/lua-packages.nix
index d120e9f7a7f..d2410e2eef3 100644
--- a/pkgs/top-level/lua-packages.nix
+++ b/pkgs/top-level/lua-packages.nix
@@ -134,13 +134,13 @@ with self; {
 
   vicious = toLuaModule(stdenv.mkDerivation rec {
     pname = "vicious";
-    version = "2.3.1";
+    version = "2.5.0";
 
     src = fetchFromGitHub {
       owner = "Mic92";
       repo = "vicious";
       rev = "v${version}";
-      sha256 = "1yzhjn8rsvjjsfycdc993ms6jy2j5jh7x3r2ax6g02z5n0anvnbx";
+      sha256 = "0lb90334mz0my8ydsmnsnkki0xr58kinsg0hf9d6k4b0vjfi0r0a";
     };
 
     buildInputs = [ lua ];

From 66c16e12fa0ba419695369df397e854aa7a3a7d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Tue, 8 Dec 2020 12:10:47 +0100
Subject: [PATCH 05/26] buildFhsUserenv: don't downgrade root user

---
 .../build-fhs-userenv/chrootenv/chrootenv.c      | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/pkgs/build-support/build-fhs-userenv/chrootenv/chrootenv.c b/pkgs/build-support/build-fhs-userenv/chrootenv/chrootenv.c
index dcb2e97aa93..a438b80e182 100644
--- a/pkgs/build-support/build-fhs-userenv/chrootenv/chrootenv.c
+++ b/pkgs/build-support/build-fhs-userenv/chrootenv/chrootenv.c
@@ -43,6 +43,7 @@ const gchar *create_tmpdir() {
 void pivot_host(const gchar *guest) {
   g_autofree gchar *point = g_build_filename(guest, "host", NULL);
   fail_if(g_mkdir(point, 0755));
+  fail_if(mount(0, "/", 0, MS_PRIVATE | MS_REC, 0));
   fail_if(pivot_root(guest, point));
 }
 
@@ -56,6 +57,7 @@ void bind_mount_item(const gchar *host, const gchar *guest, const gchar *name) {
 
 void bind(const gchar *host, const gchar *guest) {
   mount_tmpfs(guest);
+
   pivot_host(guest);
 
   g_autofree gchar *host_dir = g_build_filename("/host", host, NULL);
@@ -105,7 +107,11 @@ int main(gint argc, gchar **argv) {
     uid_t uid = getuid();
     gid_t gid = getgid();
 
-    if (unshare(CLONE_NEWNS | CLONE_NEWUSER) < 0) {
+    int namespaces = CLONE_NEWNS;
+    if (uid != 0) {
+      namespaces |= CLONE_NEWUSER;
+    }
+    if (unshare(namespaces) < 0) {
       int unshare_errno = errno;
 
       g_message("Requires Linux version >= 3.19 built with CONFIG_USER_NS");
@@ -116,9 +122,11 @@ int main(gint argc, gchar **argv) {
       fail("unshare", unshare_errno);
     }
 
-    spit("/proc/self/setgroups", "deny");
-    spit("/proc/self/uid_map", "%d %d 1", uid, uid);
-    spit("/proc/self/gid_map", "%d %d 1", gid, gid);
+    if (uid != 0) {
+      spit("/proc/self/setgroups", "deny");
+      spit("/proc/self/uid_map", "%d %d 1", uid, uid);
+      spit("/proc/self/gid_map", "%d %d 1", gid, gid);
+    }
 
     // If there is a /host directory, assume this is nested chrootenv and use it as host instead.
     gboolean nested_host = g_file_test("/host", G_FILE_TEST_EXISTS | G_FILE_TEST_IS_DIR);

From 09a76e08aead9ea1b466ad512482f88b6d64d57a Mon Sep 17 00:00:00 2001
From: Michael Roitzsch <reactorcontrol@icloud.com>
Date: Sun, 27 Dec 2020 19:04:59 +0100
Subject: [PATCH 06/26] glibc: fix Darwin cross compilation for 2.32

---
 .../libraries/glibc/darwin-cross-build.patch  | 44 ++++---------------
 1 file changed, 9 insertions(+), 35 deletions(-)

diff --git a/pkgs/development/libraries/glibc/darwin-cross-build.patch b/pkgs/development/libraries/glibc/darwin-cross-build.patch
index 7b224924104..b746e6369a0 100644
--- a/pkgs/development/libraries/glibc/darwin-cross-build.patch
+++ b/pkgs/development/libraries/glibc/darwin-cross-build.patch
@@ -3,10 +3,8 @@ enable cross-compilation of glibc on Darwin (build=Darwin, host=Linux)
 * use host version of ar, which is given by environment variable
 * build system uses stamp.os and stamp.oS files, which only differ in case;
   this fails on macOS, so replace .oS with .o_S
-* libintl.h does not exist (and is not needed) on macOS
-
---- glibc-2.27/Makefile.in	2018-02-01 17:17:18.000000000 +0100
-+++ glibc-2.27/Makefile.in	2019-02-15 17:38:27.022965553 +0100
+--- glibc-2.32/Makefile.in	2018-02-01 17:17:18.000000000 +0100
++++ glibc-2.32/Makefile.in	2020-12-27 18:21:30.000000000 +0100
 @@ -6,9 +6,11 @@
  .PHONY: all install bench
  
@@ -20,9 +18,9 @@ enable cross-compilation of glibc on Darwin (build=Darwin, host=Linux)
  	LC_ALL=C; export LC_ALL; \
  	$(MAKE) -r PARALLELMFLAGS="$(PARALLELMFLAGS)" -C $(srcdir) objdir=`pwd` $@
  
---- glibc-2.27/Makerules	2018-02-01 17:17:18.000000000 +0100
-+++ glibc-2.27/Makerules	2019-02-15 17:43:11.196039000 +0100
-@@ -915,8 +915,8 @@
+--- glibc-2.32/Makerules	2018-02-01 17:17:18.000000000 +0100
++++ glibc-2.32/Makerules	2020-12-27 18:21:30.000000000 +0100
+@@ -847,8 +847,8 @@
  ifndef objects
  
  # Create the stamp$o files to keep the parent makefile happy.
@@ -33,7 +31,7 @@ enable cross-compilation of glibc on Darwin (build=Darwin, host=Linux)
  	$(make-target-directory)
  	rm -f $@; > $@
  else
-@@ -927,7 +927,7 @@
+@@ -859,7 +859,7 @@
  # The parent will then actually add them all to the archive in the
  # archive rule, below.
  define o-iterator-doit
@@ -42,7 +40,7 @@ enable cross-compilation of glibc on Darwin (build=Darwin, host=Linux)
  endef
  define do-stamp
  $(make-target-directory)
-@@ -943,14 +943,14 @@
+@@ -875,14 +875,14 @@
  # on the stamp files built above.
  define o-iterator-doit
  $(common-objpfx)$(patsubst %,$(libtype$o),c): \
@@ -59,7 +57,7 @@ enable cross-compilation of glibc on Darwin (build=Darwin, host=Linux)
  ifndef subdir
  $(subdirs-stamps): subdir_lib;
  endif
-@@ -961,7 +961,7 @@
+@@ -893,7 +893,7 @@
  # This makes all the object files.
  .PHONY: objects objs libobjs extra-objs
  objects objs: libobjs extra-objs
@@ -68,7 +66,7 @@ enable cross-compilation of glibc on Darwin (build=Darwin, host=Linux)
  extra-objs: $(addprefix $(objpfx),$(extra-objs))
  
  # Canned sequence for building an extra library archive.
-@@ -1615,7 +1615,7 @@
+@@ -1499,7 +1499,7 @@
  	$(rmobjs)
  define rmobjs
  $(foreach o,$(object-suffixes-for-libc),
@@ -77,27 +75,3 @@ enable cross-compilation of glibc on Darwin (build=Darwin, host=Linux)
  endef
  
  # Also remove the dependencies and generated source files.
---- glibc-2.27/sunrpc/rpc_main.c	2019-02-15 17:32:43.710244513 +0100
-+++ glibc-2.27/sunrpc/rpc_main.c	2019-02-15 17:23:57.139617796 +0100
-@@ -38,7 +38,9 @@
- #include <stdio.h>
- #include <string.h>
- #include <unistd.h>
-+#ifndef __APPLE__
- #include <libintl.h>
-+#endif
- #include <locale.h>
- #include <ctype.h>
- #include <sys/types.h>
---- glibc-2.27/sunrpc/rpc_scan.c	2019-02-15 17:32:54.845490606 +0100
-+++ glibc-2.27/sunrpc/rpc_scan.c	2019-02-15 17:24:54.288066644 +0100
-@@ -37,7 +37,9 @@
- #include <stdio.h>
- #include <ctype.h>
- #include <string.h>
-+#ifndef __APPLE__
- #include <libintl.h>
-+#endif
- #include "rpc_scan.h"
- #include "rpc_parse.h"
- #include "rpc_util.h"

From a629257ec5e28992ad92316362a4f4662a3001a4 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Thu, 24 Dec 2020 02:52:32 +0100
Subject: [PATCH 07/26] tree-sitter: rename library.nix to grammar.nix

---
 pkgs/development/tools/parsing/tree-sitter/default.nix          | 2 +-
 .../tools/parsing/tree-sitter/{library.nix => grammar.nix}      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename pkgs/development/tools/parsing/tree-sitter/{library.nix => grammar.nix} (94%)

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index 57c12cce93e..2cfa6398113 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -43,7 +43,7 @@ let
 
   builtGrammars = let
     change = name: grammar:
-      callPackage ./library.nix {} {
+      callPackage ./grammar.nix {} {
         language = name;
         inherit version;
         source = fetchGrammar grammar;
diff --git a/pkgs/development/tools/parsing/tree-sitter/library.nix b/pkgs/development/tools/parsing/tree-sitter/grammar.nix
similarity index 94%
rename from pkgs/development/tools/parsing/tree-sitter/library.nix
rename to pkgs/development/tools/parsing/tree-sitter/grammar.nix
index f78939b22ff..f92e0d79426 100644
--- a/pkgs/development/tools/parsing/tree-sitter/library.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/grammar.nix
@@ -15,7 +15,7 @@
 
 stdenv.mkDerivation {
 
-  pname = "tree-sitter-${language}-library";
+  pname = "${language}-grammar";
   inherit version;
 
   src = source;

From c0a4b41afeb066b0d5eb6c0adbf6be6244966a88 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Thu, 24 Dec 2020 03:01:10 +0100
Subject: [PATCH 08/26] tree-sitter: improve update script to fetch all
 available grammars
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The update script would only fetch the few grammars listed in the
tree-sitter repository previously. But the tree-sitter github orga has
a rather large amount of officially supported grammars.

Thus we change the script to query the github APIs for repositories
instead (up to 100 this is supported without paging).

Since the repository list also contains some that are not grammars,
there is a bash script which lists all repos we are aware of and the
ones we want to ignore. It will make sure we don’t forget any
repositories in the future, by comparing to the actual list with jq.
---
 .../tools/parsing/tree-sitter/update.nix      | 94 +++++++++++++++++--
 1 file changed, 86 insertions(+), 8 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 2a3575a44d2..430d2ead026 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -3,11 +3,70 @@
 , src }:
 
 let
-  # print all the grammar names mentioned in the fetch-fixtures script
-  getGrammarNames = writeShellScript "get-grammars.sh" ''
+  # check in the list of grammars, whether we know all of them.
+  checkKnownGrammars = writeShellScript "get-grammars.sh" ''
     set -euo pipefail
-    sed -ne 's/^fetch_grammar \(\S*\).*$/\1/p' \
-      ${src}/script/fetch-fixtures
+    known='
+    [ "tree-sitter-javascript"
+    , "tree-sitter-c"
+    , "tree-sitter-swift"
+    , "tree-sitter-json"
+    , "tree-sitter-cpp"
+    , "tree-sitter-ruby"
+    , "tree-sitter-razor"
+    , "tree-sitter-go"
+    , "tree-sitter-c-sharp"
+    , "tree-sitter-python"
+    , "tree-sitter-typescript"
+    , "tree-sitter-rust"
+    , "tree-sitter-bash"
+    , "tree-sitter-php"
+    , "tree-sitter-java"
+    , "tree-sitter-scala"
+    , "tree-sitter-ocaml"
+    , "tree-sitter-julia"
+    , "tree-sitter-agda"
+    , "tree-sitter-fluent"
+    , "tree-sitter-html"
+    , "tree-sitter-haskell"
+    , "tree-sitter-regex"
+    , "tree-sitter-css"
+    , "tree-sitter-verilog"
+    , "tree-sitter-jsdoc"
+    , "tree-sitter-ql"
+    ]'
+    ignore='
+    [ "tree-sitter"
+    , "tree-sitter-cli"
+    , "tree-sitter-embedded-template"
+    ${/*this is the haskell language bindings, tree-sitter-haskell is the grammar*/""}
+    , "haskell-tree-sitter"
+    ${/*this is the ruby language bindings, tree-sitter-ruby is the grammar*/""}
+    , "ruby-tree-sitter"
+    ${/*this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar*/""}
+    , "rust-tree-sitter"
+    ${/*this is the nodejs language bindings, tree-sitter-javascript is the grammar*/""}
+    , "node-tree-sitter"
+    ${/*this is the python language bindings, tree-sitter-python is the grammar*/""}
+    , "py-tree-sitter"
+    ${/*afl fuzzing for tree sitter*/""}
+    , "afl-tree-sitter"
+    ${/*archived*/""}
+    , "highlight-schema"
+    ${/*website*/""}
+    , "tree-sitter.github.io"
+    ]'
+    res=$(${jq}/bin/jq \
+      --argjson known "$known" \
+      --argjson ignore "$ignore" \
+      '. - ($known + $ignore)' \
+      )
+    if [ ! "$res" == "[]" ]; then
+      echo "These repositories are neither known nor ignored:" 1>&2
+      echo "$res" 1>&2
+      exit 1
+    fi
+    printf '%s' "$known"
   '';
 
   # TODO
@@ -22,7 +81,7 @@ let
     res=$(${curl}/bin/curl \
       --silent \
       "https://api.github.com/repos/${urlEscape owner}/$(${urlEscapeSh} "$repo")/releases/latest")
-    if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message')" =~ "rate limit" ]]; then
+    if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
       echo "rate limited" >&2
     fi
     release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name')
@@ -34,6 +93,21 @@ let
     echo "$release"
   '';
 
+  # find the latest repos of a github organization
+  latestGithubRepos = { orga }: writeShellScript "latest-github-repos" ''
+    set -euo pipefail
+    res=$(${curl}/bin/curl \
+      --silent \
+      'https://api.github.com/orgs/${orga}/repos?per_page=100')
+
+    if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
+      echo "rate limited" >&2
+    fi
+
+    printf "%s" "$res" | ${jq}/bin/jq 'map(.name)' \
+      || echo "failed $res"
+  '';
+
   # update one tree-sitter grammar repo and print their nix-prefetch-git output
   updateGrammar = { owner }: writeShellScript "update-grammar.sh" ''
     set -euo pipefail
@@ -49,18 +123,22 @@ let
 
   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
     set -euo pipefail
-    grammarNames=$(${getGrammarNames})
+    echo "fetching list of grammars" 1>&2
+    grammars=$(${latestGithubRepos { orga = "tree-sitter"; }})
+    echo "checking against the list of grammars we know" 1>&2
+    knownGrammars=$(printf '%s' "$grammars" | ${checkKnownGrammars})
+    # change the json list into a item-per-line bash format
+    grammarNames=$(printf '%s' "$knownGrammars" | ${jq}/bin/jq --raw-output '.[]')
     outputDir="${toString ./.}/grammars"
     mkdir -p "$outputDir"
     updateCommand=$(printf \
       '${updateGrammar { owner = "tree-sitter"; }} "$1" > "%s/$1.json"' \
       "$outputDir")
     printf '%s' "$grammarNames" \
-      | ${xe}/bin/xe printf "tree-sitter-%s\n" {} \
       | ${xe}/bin/xe -j2 -s "$updateCommand"
     ( echo "{"
       printf '%s' "$grammarNames" \
-        | ${xe}/bin/xe -s 'printf "  %s = (builtins.fromJSON (builtins.readFile ./tree-sitter-%s.json));\n" "$1" "$1"'
+        | ${xe}/bin/xe -s 'printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "$1" "$1"'
       echo "}" ) \
       > "$outputDir/default.nix"
   '';

From a32c5d90e7a4c4cc32088b30e591b516fe761952 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Thu, 24 Dec 2020 03:09:25 +0100
Subject: [PATCH 09/26] tree-sitter: update grammars

The new update scripts gives us a bunch of new grammars!
---
 .../tools/parsing/tree-sitter/default.nix     | 12 ++++-
 .../parsing/tree-sitter/grammars/default.nix  | 44 ++++++++++++-------
 .../grammars/tree-sitter-agda.json            | 10 +++++
 .../grammars/tree-sitter-c-sharp.json         | 10 +++++
 .../tree-sitter/grammars/tree-sitter-cpp.json |  8 ++--
 .../tree-sitter/grammars/tree-sitter-css.json | 10 +++++
 .../grammars/tree-sitter-fluent.json          | 10 +++++
 .../grammars/tree-sitter-haskell.json         | 10 +++++
 .../grammars/tree-sitter-javascript.json      |  8 ++--
 .../grammars/tree-sitter-julia.json           | 10 +++++
 .../grammars/tree-sitter-ocaml.json           | 10 +++++
 .../grammars/tree-sitter-python.json          |  8 ++--
 .../tree-sitter/grammars/tree-sitter-ql.json  | 10 +++++
 .../grammars/tree-sitter-razor.json           | 10 +++++
 .../grammars/tree-sitter-regex.json           | 10 +++++
 .../grammars/tree-sitter-rust.json            |  8 ++--
 .../grammars/tree-sitter-scala.json           | 10 +++++
 .../grammars/tree-sitter-swift.json           | 10 +++++
 .../grammars/tree-sitter-typescript.json      |  8 ++--
 .../grammars/tree-sitter-verilog.json         | 10 +++++
 .../tools/parsing/tree-sitter/update.nix      |  2 +-
 21 files changed, 189 insertions(+), 39 deletions(-)
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json
 create mode 100644 pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index 2cfa6398113..38d8beb90db 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -49,8 +49,16 @@ let
         source = fetchGrammar grammar;
       };
   in
-    # typescript doesn't have parser.c in the same place as others
-    lib.mapAttrs change (removeAttrs (import ./grammars) ["typescript"]);
+    lib.mapAttrs change (removeAttrs (import ./grammars) [
+      # TODO these don't have parser.c in the same place as others.
+      # They might require more elaborate builds?
+      #  /nix/…/src/parser.c: No such file or directory
+      "tree-sitter-typescript"
+      #  /nix/…/src/parser.c: No such file or directory
+      "tree-sitter-ocaml"
+      # /nix/…/src/parser.c:1:10: fatal error: tree_sitter/parser.h: No such file or directory
+      "tree-sitter-razor"
+    ]);
 
 in rustPlatform.buildRustPackage {
   pname = "tree-sitter";
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
index e9cb0a1db6e..18217f69a5b 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
@@ -1,18 +1,30 @@
 {
-  bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
-  c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
-  cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
-  embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
-  go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
-  html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
-  java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
-  javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
-  jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
-  json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
-  lua = (builtins.fromJSON (builtins.readFile ./tree-sitter-lua.json));
-  php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
-  python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
-  ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
-  rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
-  typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
+  tree-sitter-javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
+  tree-sitter-c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
+  tree-sitter-swift = (builtins.fromJSON (builtins.readFile ./tree-sitter-swift.json));
+  tree-sitter-json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
+  tree-sitter-cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
+  tree-sitter-ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
+  tree-sitter-razor = (builtins.fromJSON (builtins.readFile ./tree-sitter-razor.json));
+  tree-sitter-go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
+  tree-sitter-c-sharp = (builtins.fromJSON (builtins.readFile ./tree-sitter-c-sharp.json));
+  tree-sitter-python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
+  tree-sitter-typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
+  tree-sitter-rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
+  tree-sitter-bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
+  tree-sitter-php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
+  tree-sitter-java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
+  tree-sitter-scala = (builtins.fromJSON (builtins.readFile ./tree-sitter-scala.json));
+  tree-sitter-ocaml = (builtins.fromJSON (builtins.readFile ./tree-sitter-ocaml.json));
+  tree-sitter-julia = (builtins.fromJSON (builtins.readFile ./tree-sitter-julia.json));
+  tree-sitter-agda = (builtins.fromJSON (builtins.readFile ./tree-sitter-agda.json));
+  tree-sitter-fluent = (builtins.fromJSON (builtins.readFile ./tree-sitter-fluent.json));
+  tree-sitter-html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
+  tree-sitter-haskell = (builtins.fromJSON (builtins.readFile ./tree-sitter-haskell.json));
+  tree-sitter-regex = (builtins.fromJSON (builtins.readFile ./tree-sitter-regex.json));
+  tree-sitter-css = (builtins.fromJSON (builtins.readFile ./tree-sitter-css.json));
+  tree-sitter-verilog = (builtins.fromJSON (builtins.readFile ./tree-sitter-verilog.json));
+  tree-sitter-jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
+  tree-sitter-ql = (builtins.fromJSON (builtins.readFile ./tree-sitter-ql.json));
+  tree-sitter-embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
 }
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json
new file mode 100644
index 00000000000..f16877e9465
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-agda.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-agda",
+  "rev": "d710ff14d15ddee3764fd73a0837c6c4c8c913e9",
+  "date": "2019-09-20T18:06:06+08:00",
+  "path": "/nix/store/wqz9v9znaiwhhqi19hgig9bn0yvl4i9s-tree-sitter-agda",
+  "sha256": "1wpfj47l97pxk3i9rzdylqipy849r482fnj3lmx8byhalv7z1vm6",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json
new file mode 100644
index 00000000000..6498e40497e
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-c-sharp.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-c-sharp",
+  "rev": "b8bff2a389ecc098dbb7e8abee492816a5eb42db",
+  "date": "2020-11-15T07:54:17+00:00",
+  "path": "/nix/store/h5p71g661hbyzcdcj6xff8y5pcsivpa4-tree-sitter-c-sharp",
+  "sha256": "0x78s2wgd8b6pwjzbmc9fgp0ivdmxv39wikig1m55slai6yq51wh",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json
index 16aef8ee33e..a4538b1a932 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-cpp.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-cpp",
-  "rev": "fb8250eef8b4cf1ce104806c50dc206e388b0e72",
-  "date": "2020-11-02T09:46:58-08:00",
-  "path": "/nix/store/qmza43f0fraa111bg50vaxsgnrs5kqs1-tree-sitter-cpp",
-  "sha256": "1sbk5a2p1fh6798naiswsap6fpj7n7bqhwd3xdyi35xf613qpwsi",
+  "rev": "a35a275df92e7583df38f2de2562361f2b69987e",
+  "date": "2020-12-13T11:27:21-08:00",
+  "path": "/nix/store/l0mv4q1xdxz94ym1nl73y52i1yr9zcgi-tree-sitter-cpp",
+  "sha256": "130vizybkm11j3lpzmf183myz0vjxq75mpy6qz48rrkidhnrlryk",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json
new file mode 100644
index 00000000000..b8609c0bd17
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-css.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-css",
+  "rev": "23f2cb97d47860c517f67f03e1f4b621d5bd2085",
+  "date": "2020-05-14T14:44:30-07:00",
+  "path": "/nix/store/r5pkz9kly0mhgrmqzdzdsr6d1dpqavld-tree-sitter-css",
+  "sha256": "17svpf36p0p7spppzhm3fi833zpdl2l1scg34r6d4vcbv7dknrjy",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json
new file mode 100644
index 00000000000..f40b8465b7d
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-fluent.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-fluent",
+  "rev": "858fdd6f1e81992e00d3541bfb31bac9365d7a47",
+  "date": "2018-06-18T13:00:38-07:00",
+  "path": "/nix/store/zbj8abdlrqi9swm8qn8rhpqmjwcz145f-tree-sitter-fluent",
+  "sha256": "0528v9w0cs73p9048xrddb1wpdhr92sn1sw8yyqfrq5sq0danr9k",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json
new file mode 100644
index 00000000000..10d2dd64ac2
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-haskell.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-haskell",
+  "rev": "2a0aa1cb5f1b787a4056a29fa0791e87846e33fb",
+  "date": "2018-11-03T09:56:20-07:00",
+  "path": "/nix/store/9xszs4xi51qr72laxr67zxnh8y2br0gy-tree-sitter-haskell",
+  "sha256": "0z0nfip5m0yrjgm49j5nld7jkgyjdmps101xhbng39pwfnh1av83",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json
index 68008d393d1..2c563f3fbfa 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-javascript.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-javascript",
-  "rev": "852f11b394804ac2a8986f8bcaafe77753635667",
-  "date": "2020-10-27T13:43:40-04:00",
-  "path": "/nix/store/7a9nd3prxbv2izvilqdxf58his097ak7-tree-sitter-javascript",
-  "sha256": "1cdqx75fm7fvna0iymw7n03a7f6gayfg97qwqz3himi0775fz9ir",
+  "rev": "3f8b62f9befd3cb3b4cb0de22f6595a0aadf76ca",
+  "date": "2020-12-02T10:20:20-08:00",
+  "path": "/nix/store/c17bf7sjq95lank5ygbglv8j48i5z9w3-tree-sitter-javascript",
+  "sha256": "0fjq1jzrzd8c8rfxkh2s25gnqlyc19k3a8i3r1129kakisn1288k",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json
new file mode 100644
index 00000000000..2cf5f06dc19
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-julia.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-julia",
+  "rev": "6a0863f1ce3fcf6f99dc0addb7886dcbd27c5a48",
+  "date": "2020-09-08T19:39:52-07:00",
+  "path": "/nix/store/xn5nii9mi2aw7xdabyxlglha2vk12h1w-tree-sitter-julia",
+  "sha256": "07ds4wzgvnkgkq07izdglkk8lgpqrylvrs96afnwxi56bnzs8sbv",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json
new file mode 100644
index 00000000000..32488ebb73e
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ocaml.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-ocaml",
+  "rev": "4ec9ee414dadc2b6e7325a9f8124d02f6cd8c250",
+  "date": "2020-09-18T02:13:40+02:00",
+  "path": "/nix/store/dszjdnwnbziqxav2khs85026msm6fasz-tree-sitter-ocaml",
+  "sha256": "0wy85940fhmrnz7c1gk6xkipm8ixzalq5q4i7mcc6wnjiiwq60gx",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json
index 402c4c087b1..86551acbee0 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-python.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-python",
-  "rev": "d245768132eb6cb74d8a394ca4d29dc57169b096",
-  "date": "2020-10-27T13:20:04-04:00",
-  "path": "/nix/store/afv7ibxpf5c2shafa89cnza88hsfb7kj-tree-sitter-python",
-  "sha256": "075r5i2id2rn76xm3pcrn5cpvj63dlaxcpvphig39a4c9f9hrpdx",
+  "rev": "65b486b42fe0188e4be344092151042bf1bc6d9a",
+  "date": "2020-12-15T09:33:20-08:00",
+  "path": "/nix/store/k9jziqzyxq2bv55pwl03jcmmca83fjyp-tree-sitter-python",
+  "sha256": "1yxqdlmp0jybm7vvza1ni5a320vrviqkd14pnpcrg9ilzq23mlsh",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json
new file mode 100644
index 00000000000..27042b6ef95
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ql.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-ql",
+  "rev": "a0d688d62dcb9cbc7c53f0d98343c458b3776b3d",
+  "date": "2020-09-16T12:56:09-07:00",
+  "path": "/nix/store/dfdaf6wg80dfw5fvdiir7n9nj6j30g3g-tree-sitter-ql",
+  "sha256": "0f6rfhrbvpg8czfa7mld45by3rp628bs6fyl47a8mn18w6x0n5g2",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json
new file mode 100644
index 00000000000..6905f4145aa
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-razor.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-razor",
+  "rev": "60edbd8e798e416f5226a746396efa6a8614fa9b",
+  "date": "2016-07-08T15:17:50-07:00",
+  "path": "/nix/store/2vkxw48wc0xf07awi65r4rg2m1lpqq6c-tree-sitter-razor",
+  "sha256": "1ldsn9nxldxkxdy7irywx88d6a56q05f6907lypzass07piyqp2i",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json
new file mode 100644
index 00000000000..d04c1bdab86
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-regex.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-regex",
+  "rev": "be2e415b5716615530234d179dc27c32b7a1d86b",
+  "date": "2020-05-14T15:05:54-07:00",
+  "path": "/nix/store/acbcq9lgh5pf6hq3329fa253yfvaj6ji-tree-sitter-regex",
+  "sha256": "0qxbafbwfj64bwhj3455jgkidpdzqwrlsqm3rckq2pi75flnkv42",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json
index 4e0e2e2f7f4..38c9e44f2f6 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-rust.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-rust",
-  "rev": "cf47ff80cbd8bdc6dd649a005c5f5d73f9c347e2",
-  "date": "2020-11-04T06:23:40-08:00",
-  "path": "/nix/store/8cvjxw8cz5jrkqwvgq88r7rfm269xwk1-tree-sitter-rust",
-  "sha256": "0yzcag0yy3ncxwpnb3fmsw8bxaidp7z8kbl8wwaxaj2vvxxf6sam",
+  "rev": "8746bd4b584b8063ee8e445bf31015e887417d33",
+  "date": "2020-11-23T17:16:42-08:00",
+  "path": "/nix/store/1zzxvza23wsdyazw47lhjvrs0za6wjpn-tree-sitter-rust",
+  "sha256": "0bhxfyq8ycnp90pqvr6cf3gpq1vax8a34kaq85dmbrc6ar8a7ap6",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json
new file mode 100644
index 00000000000..8d9959cf4cf
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-scala.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-scala",
+  "rev": "211bb726bb5857f872247b600c7c1808e641a8d4",
+  "date": "2020-07-13T13:31:00-07:00",
+  "path": "/nix/store/6q66gzabxazr2581dgp4pp5wwyk2p6mw-tree-sitter-scala",
+  "sha256": "096ps5za8gxmq61gdd3xdk8cif07vb4v8asls2kdwm6jazm82777",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json
new file mode 100644
index 00000000000..8f73380e379
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-swift.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-swift",
+  "rev": "a22fa5e19bae50098e2252ea96cba3aba43f4c58",
+  "date": "2019-10-24T19:04:02-06:00",
+  "path": "/nix/store/pk5xk8yp6vanbar75bhfrs104w0k1ph0-tree-sitter-swift",
+  "sha256": "14b40lmwrnyvdz2wiv684kfh4fvqfhbj1dgrx81ppmy7hsz7jcq7",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json
index d31f72754bf..a875232eab6 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-typescript.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-typescript",
-  "rev": "73afadbd117a8e8551758af9c3a522ef46452119",
-  "date": "2020-10-20T12:15:52-07:00",
-  "path": "/nix/store/wrgp1j5l50cigv5cmlxikw693a55i6g9-tree-sitter-typescript",
-  "sha256": "0dsbmcvjzys9s229drp1l8anram7d6nsx1nixl5m9znw54hr4w9p",
+  "rev": "a3a4becef889692724bb1c2191f8fc4bce6573f9",
+  "date": "2020-12-15T13:34:55-08:00",
+  "path": "/nix/store/dra5wbhgxkkhphm1mhgv87hsfw0g18nw-tree-sitter-typescript",
+  "sha256": "0ir7lmzwnbf1zwbl4bfsib1jilg8pvfc1nabaq2n2rixvghlvf3d",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json
new file mode 100644
index 00000000000..7ab79c6f2d5
--- /dev/null
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-verilog.json
@@ -0,0 +1,10 @@
+{
+  "url": "https://github.com/tree-sitter/tree-sitter-verilog",
+  "rev": "ad551aae2649da56582bc0557478a7dc979c0be3",
+  "date": "2020-10-13T17:40:47-07:00",
+  "path": "/nix/store/nfaxfqrqkxpwaq8rnk7kcp28nnj8y6m2-tree-sitter-verilog",
+  "sha256": "0cy29i200rnc34d237s19r6a1n5vv4d3wgwpbywxg6ahcankc34m",
+  "fetchSubmodules": false,
+  "deepClone": false,
+  "leaveDotGit": false
+}
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 430d2ead026..8368857cb8a 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -34,11 +34,11 @@ let
     , "tree-sitter-verilog"
     , "tree-sitter-jsdoc"
     , "tree-sitter-ql"
+    , "tree-sitter-embedded-template"
     ]'
     ignore='
     [ "tree-sitter"
     , "tree-sitter-cli"
-    , "tree-sitter-embedded-template"
     ${/*this is the haskell language bindings, tree-sitter-haskell is the grammar*/""}
     , "haskell-tree-sitter"
     ${/*this is the ruby language bindings, tree-sitter-ruby is the grammar*/""}

From d2988dac8eb1f91a522adc35c8594260e06ea8b4 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Sat, 26 Dec 2020 20:11:43 +0100
Subject: [PATCH 10/26] tree-sitter/update: factor repo list into nix

We want the ability to add different orga repos as well, and that is a
lot easier on the nix level.
---
 .../tools/parsing/tree-sitter/default.nix     |   4 +-
 .../tools/parsing/tree-sitter/update.nix      | 118 ++++++++++--------
 2 files changed, 65 insertions(+), 57 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index 38d8beb90db..ee0c870722e 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -1,6 +1,6 @@
 { lib, stdenv
 , fetchgit, fetchFromGitHub, fetchurl
-, writeShellScript, runCommand, which
+, writeShellScript, runCommand, which, formats
 , rustPlatform, jq, nix-prefetch-git, xe, curl, emscripten
 , Security
 , callPackage
@@ -29,7 +29,7 @@ let
   };
 
   update-all-grammars = import ./update.nix {
-    inherit writeShellScript nix-prefetch-git curl jq xe src;
+    inherit writeShellScript nix-prefetch-git curl jq xe src formats;
   };
 
   fetchGrammar = (v: fetchgit {inherit (v) url rev sha256 fetchSubmodules; });
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 8368857cb8a..cb08969d296 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -1,72 +1,80 @@
-{ writeShellScript, nix-prefetch-git
+{ writeShellScript, nix-prefetch-git, formats
 , curl, jq, xe
 , src }:
 
 let
+  # Grammars we want to fetch from the tree-sitter github orga
+  knownTreeSitterOrgGrammarRepos = [
+    "tree-sitter-javascript"
+    "tree-sitter-c"
+    "tree-sitter-swift"
+    "tree-sitter-json"
+    "tree-sitter-cpp"
+    "tree-sitter-ruby"
+    "tree-sitter-razor"
+    "tree-sitter-go"
+    "tree-sitter-c-sharp"
+    "tree-sitter-python"
+    "tree-sitter-typescript"
+    "tree-sitter-rust"
+    "tree-sitter-bash"
+    "tree-sitter-php"
+    "tree-sitter-java"
+    "tree-sitter-scala"
+    "tree-sitter-ocaml"
+    "tree-sitter-julia"
+    "tree-sitter-agda"
+    "tree-sitter-fluent"
+    "tree-sitter-html"
+    "tree-sitter-haskell"
+    "tree-sitter-regex"
+    "tree-sitter-css"
+    "tree-sitter-verilog"
+    "tree-sitter-jsdoc"
+    "tree-sitter-ql"
+    "tree-sitter-embedded-template"
+  ];
+
+  # repos of the tree-sitter github orga we want to ignore (not grammars)
+  ignoredTreeSitterOrgRepos = [
+    "tree-sitter"
+    "tree-sitter-cli"
+    # this is the haskell language bindings, tree-sitter-haskell is the grammar
+    "haskell-tree-sitter"
+    # this is the ruby language bindings, tree-sitter-ruby is the grammar
+    "ruby-tree-sitter"
+    # this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar
+    "rust-tree-sitter"
+    # this is the nodejs language bindings, tree-sitter-javascript is the grammar
+    "node-tree-sitter"
+    # this is the python language bindings, tree-sitter-python is the grammar
+    "py-tree-sitter"
+    # afl fuzzing for tree sitter
+    "afl-tree-sitter"
+    # archived
+    "highlight-schema"
+    # website
+    "tree-sitter.github.io"
+  ];
+
+  jsonFile = name: val: (formats.json {}).generate name val;
+
   # check in the list of grammars, whether we know all of them.
   checkKnownGrammars = writeShellScript "get-grammars.sh" ''
     set -euo pipefail
-    known='
-    [ "tree-sitter-javascript"
-    , "tree-sitter-c"
-    , "tree-sitter-swift"
-    , "tree-sitter-json"
-    , "tree-sitter-cpp"
-    , "tree-sitter-ruby"
-    , "tree-sitter-razor"
-    , "tree-sitter-go"
-    , "tree-sitter-c-sharp"
-    , "tree-sitter-python"
-    , "tree-sitter-typescript"
-    , "tree-sitter-rust"
-    , "tree-sitter-bash"
-    , "tree-sitter-php"
-    , "tree-sitter-java"
-    , "tree-sitter-scala"
-    , "tree-sitter-ocaml"
-    , "tree-sitter-julia"
-    , "tree-sitter-agda"
-    , "tree-sitter-fluent"
-    , "tree-sitter-html"
-    , "tree-sitter-haskell"
-    , "tree-sitter-regex"
-    , "tree-sitter-css"
-    , "tree-sitter-verilog"
-    , "tree-sitter-jsdoc"
-    , "tree-sitter-ql"
-    , "tree-sitter-embedded-template"
-    ]'
-    ignore='
-    [ "tree-sitter"
-    , "tree-sitter-cli"
-    ${/*this is the haskell language bindings, tree-sitter-haskell is the grammar*/""}
-    , "haskell-tree-sitter"
-    ${/*this is the ruby language bindings, tree-sitter-ruby is the grammar*/""}
-    , "ruby-tree-sitter"
-    ${/*this is the (unmaintained) rust language bindings, tree-sitter-rust is the grammar*/""}
-    , "rust-tree-sitter"
-    ${/*this is the nodejs language bindings, tree-sitter-javascript is the grammar*/""}
-    , "node-tree-sitter"
-    ${/*this is the python language bindings, tree-sitter-python is the grammar*/""}
-    , "py-tree-sitter"
-    ${/*afl fuzzing for tree sitter*/""}
-    , "afl-tree-sitter"
-    ${/*archived*/""}
-    , "highlight-schema"
-    ${/*website*/""}
-    , "tree-sitter.github.io"
-    ]'
+    known="${jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos}"
+    ignore="${jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos}"
     res=$(${jq}/bin/jq \
-      --argjson known "$known" \
-      --argjson ignore "$ignore" \
-      '. - ($known + $ignore)' \
+      --slurpfile known "$known" \
+      --slurpfile ignore "$ignore" \
+      '. - ($known[0] + $ignore[0])' \
       )
     if [ ! "$res" == "[]" ]; then
       echo "These repositories are neither known nor ignored:" 1>&2
       echo "$res" 1>&2
       exit 1
     fi
-    printf '%s' "$known"
+    cat "$known"
   '';
 
   # TODO

From 0b1bfd7d146de19d25ee728552c8b88f74a266ca Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Sat, 26 Dec 2020 20:41:19 +0100
Subject: [PATCH 11/26] tree-sitter/update: make the repo check into a boolean
 test

---
 .../tools/parsing/tree-sitter/update.nix      | 22 +++++++++----------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index cb08969d296..57a07ee3929 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -4,7 +4,7 @@
 
 let
   # Grammars we want to fetch from the tree-sitter github orga
-  knownTreeSitterOrgGrammarRepos = [
+  knownTreeSitterOrgGrammarRepos = jsonFile "known-tree-sitter-org-grammar-repos" [
     "tree-sitter-javascript"
     "tree-sitter-c"
     "tree-sitter-swift"
@@ -36,7 +36,7 @@ let
   ];
 
   # repos of the tree-sitter github orga we want to ignore (not grammars)
-  ignoredTreeSitterOrgRepos = [
+  ignoredTreeSitterOrgRepos = jsonFile "ignored-tree-sitter-org-repos" [
     "tree-sitter"
     "tree-sitter-cli"
     # this is the haskell language bindings, tree-sitter-haskell is the grammar
@@ -59,14 +59,12 @@ let
 
   jsonFile = name: val: (formats.json {}).generate name val;
 
-  # check in the list of grammars, whether we know all of them.
-  checkKnownGrammars = writeShellScript "get-grammars.sh" ''
+  # check the tree-sitter orga repos
+  checkTreeSitterRepos = writeShellScript "get-grammars.sh" ''
     set -euo pipefail
-    known="${jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos}"
-    ignore="${jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos}"
     res=$(${jq}/bin/jq \
-      --slurpfile known "$known" \
-      --slurpfile ignore "$ignore" \
+      --slurpfile known "${knownTreeSitterOrgGrammarRepos}" \
+      --slurpfile ignore "${ignoredTreeSitterOrgRepos}" \
       '. - ($known[0] + $ignore[0])' \
       )
     if [ ! "$res" == "[]" ]; then
@@ -74,7 +72,6 @@ let
       echo "$res" 1>&2
       exit 1
     fi
-    cat "$known"
   '';
 
   # TODO
@@ -132,9 +129,10 @@ let
   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
     set -euo pipefail
     echo "fetching list of grammars" 1>&2
-    grammars=$(${latestGithubRepos { orga = "tree-sitter"; }})
-    echo "checking against the list of grammars we know" 1>&2
-    knownGrammars=$(printf '%s' "$grammars" | ${checkKnownGrammars})
+    treeSitterRepos=$(${latestGithubRepos { orga = "tree-sitter"; }})
+    echo "checking the tree-sitter repo list against the grammars we know" 1>&2
+    printf '%s' "$treeSitterRepos" | ${checkTreeSitterRepos}
+    knownGrammars=$(cat "${knownTreeSitterOrgGrammarRepos}")
     # change the json list into a item-per-line bash format
     grammarNames=$(printf '%s' "$knownGrammars" | ${jq}/bin/jq --raw-output '.[]')
     outputDir="${toString ./.}/grammars"

From b4ff78a45b29c22029d32186d4f42f28d9be7919 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Mon, 28 Dec 2020 09:50:57 +0100
Subject: [PATCH 12/26] tree-sitter/update: pull repo argument into the nix
 code

---
 .../tools/parsing/tree-sitter/default.nix     |  2 +-
 .../tools/parsing/tree-sitter/update.nix      | 53 +++++++++----------
 2 files changed, 26 insertions(+), 29 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index ee0c870722e..a70757ca15e 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -29,7 +29,7 @@ let
   };
 
   update-all-grammars = import ./update.nix {
-    inherit writeShellScript nix-prefetch-git curl jq xe src formats;
+    inherit writeShellScript nix-prefetch-git curl jq xe src formats lib;
   };
 
   fetchGrammar = (v: fetchgit {inherit (v) url rev sha256 fetchSubmodules; });
diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index 57a07ee3929..c63d363f95a 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -1,10 +1,10 @@
-{ writeShellScript, nix-prefetch-git, formats
+{ writeShellScript, nix-prefetch-git, formats, lib
 , curl, jq, xe
 , src }:
 
 let
   # Grammars we want to fetch from the tree-sitter github orga
-  knownTreeSitterOrgGrammarRepos = jsonFile "known-tree-sitter-org-grammar-repos" [
+  knownTreeSitterOrgGrammarRepos = [
     "tree-sitter-javascript"
     "tree-sitter-c"
     "tree-sitter-swift"
@@ -34,9 +34,10 @@ let
     "tree-sitter-ql"
     "tree-sitter-embedded-template"
   ];
+  knownTreeSitterOrgGrammarReposJson = jsonFile "known-tree-sitter-org-grammar-repos" knownTreeSitterOrgGrammarRepos;
 
   # repos of the tree-sitter github orga we want to ignore (not grammars)
-  ignoredTreeSitterOrgRepos = jsonFile "ignored-tree-sitter-org-repos" [
+  ignoredTreeSitterOrgRepos = [
     "tree-sitter"
     "tree-sitter-cli"
     # this is the haskell language bindings, tree-sitter-haskell is the grammar
@@ -56,6 +57,7 @@ let
     # website
     "tree-sitter.github.io"
   ];
+  ignoredTreeSitterOrgReposJson = jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos;
 
   jsonFile = name: val: (formats.json {}).generate name val;
 
@@ -63,8 +65,8 @@ let
   checkTreeSitterRepos = writeShellScript "get-grammars.sh" ''
     set -euo pipefail
     res=$(${jq}/bin/jq \
-      --slurpfile known "${knownTreeSitterOrgGrammarRepos}" \
-      --slurpfile ignore "${ignoredTreeSitterOrgRepos}" \
+      --slurpfile known "${knownTreeSitterOrgGrammarReposJson}" \
+      --slurpfile ignore "${ignoredTreeSitterOrgReposJson}" \
       '. - ($known[0] + $ignore[0])' \
       )
     if [ ! "$res" == "[]" ]; then
@@ -76,23 +78,20 @@ let
 
   # TODO
   urlEscape = x: x;
-  # TODO
-  urlEscapeSh = writeShellScript "escape-url" ''printf '%s' "$1"'';
 
   # generic bash script to find the latest github release for a repo
-  latestGithubRelease = { owner }: writeShellScript "latest-github-release" ''
+  latestGithubRelease = { owner, repo }: writeShellScript "latest-github-release" ''
     set -euo pipefail
-    repo="$1"
     res=$(${curl}/bin/curl \
       --silent \
-      "https://api.github.com/repos/${urlEscape owner}/$(${urlEscapeSh} "$repo")/releases/latest")
+      "https://api.github.com/repos/${urlEscape owner}/${urlEscape repo}/releases/latest")
     if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
       echo "rate limited" >&2
     fi
     release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name')
     # github sometimes returns an empty list even tough there are releases
     if [ "$release" = "null" ]; then
-      echo "uh-oh, latest for $repo is not there, using HEAD" >&2
+      echo "uh-oh, latest for ${owner + "/" + repo} is not there, using HEAD" >&2
       release="HEAD"
     fi
     echo "$release"
@@ -103,10 +102,10 @@ let
     set -euo pipefail
     res=$(${curl}/bin/curl \
       --silent \
-      'https://api.github.com/orgs/${orga}/repos?per_page=100')
+      'https://api.github.com/orgs/${urlEscape orga}/repos?per_page=100')
 
     if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
-      echo "rate limited" >&2
+      echo "rate limited" >&2   #
     fi
 
     printf "%s" "$res" | ${jq}/bin/jq 'map(.name)' \
@@ -114,37 +113,35 @@ let
   '';
 
   # update one tree-sitter grammar repo and print their nix-prefetch-git output
-  updateGrammar = { owner }: writeShellScript "update-grammar.sh" ''
+  updateGrammar = { owner, repo }: writeShellScript "update-grammar.sh" ''
     set -euo pipefail
-    repo="$1"
-    latest="$(${latestGithubRelease { inherit owner; }} "$repo")"
-    echo "Fetching latest release ($latest) of $repo …" >&2
+    latest="$(${latestGithubRelease { inherit owner repo; }})"
+    echo "Fetching latest release ($latest) of ${repo} …" >&2
     ${nix-prefetch-git}/bin/nix-prefetch-git \
       --quiet \
       --no-deepClone \
-      --url "https://github.com/${urlEscape owner}/$(${urlEscapeSh} "$repo")" \
+      --url "https://github.com/${urlEscape owner}/${urlEscape repo}" \
       --rev "$latest"
     '';
 
+  foreachSh = list: f: lib.concatMapStringsSep "\n" f list;
+
   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
     set -euo pipefail
     echo "fetching list of grammars" 1>&2
     treeSitterRepos=$(${latestGithubRepos { orga = "tree-sitter"; }})
     echo "checking the tree-sitter repo list against the grammars we know" 1>&2
     printf '%s' "$treeSitterRepos" | ${checkTreeSitterRepos}
-    knownGrammars=$(cat "${knownTreeSitterOrgGrammarRepos}")
-    # change the json list into a item-per-line bash format
-    grammarNames=$(printf '%s' "$knownGrammars" | ${jq}/bin/jq --raw-output '.[]')
     outputDir="${toString ./.}/grammars"
+    echo "writing files to $outputDir" 1>&2
     mkdir -p "$outputDir"
-    updateCommand=$(printf \
-      '${updateGrammar { owner = "tree-sitter"; }} "$1" > "%s/$1.json"' \
-      "$outputDir")
-    printf '%s' "$grammarNames" \
-      | ${xe}/bin/xe -j2 -s "$updateCommand"
+    ${foreachSh knownTreeSitterOrgGrammarRepos
+      (repo: ''${updateGrammar { owner = "tree-sitter"; inherit repo; }} > $outputDir/${repo}.json'')}
     ( echo "{"
-      printf '%s' "$grammarNames" \
-        | ${xe}/bin/xe -s 'printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "$1" "$1"'
+      ${foreachSh knownTreeSitterOrgGrammarRepos
+        (repo: ''
+           # indentation hack
+             printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "${repo}" "${repo}"'')}
       echo "}" ) \
       > "$outputDir/default.nix"
   '';

From 5f0ace20af45f8c894380d17fae66bca5bfb2f5b Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Tue, 29 Dec 2020 12:57:09 +0100
Subject: [PATCH 13/26] tree-sitter/update: allow grammars to be fetched from
 other orgas

Some of the grammars are not in the official orga, like
tree-sitter-lua, so we make sure the updater knows how to fetch them.
---
 .../tools/parsing/tree-sitter/update.nix      | 65 +++++++++++++++----
 1 file changed, 53 insertions(+), 12 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/update.nix b/pkgs/development/tools/parsing/tree-sitter/update.nix
index c63d363f95a..7b0b16b403f 100644
--- a/pkgs/development/tools/parsing/tree-sitter/update.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/update.nix
@@ -59,6 +59,45 @@ let
   ];
   ignoredTreeSitterOrgReposJson = jsonFile "ignored-tree-sitter-org-repos" ignoredTreeSitterOrgRepos;
 
+  # Additional grammars that are not in the official github orga.
+  # If you need a grammar that already exists in the official orga,
+  # make sure to give it a different name.
+  otherGrammars = {
+    "tree-sitter-lua" = {
+      orga = "nvim-treesitter";
+      repo = "tree-sitter-lua";
+    };
+  };
+
+  allGrammars =
+    let
+      treeSitterOrgaGrammars =
+        lib.listToAttrs (map (repo:
+          { name = repo;
+            value = {
+              orga = "tree-sitter";
+              inherit repo;
+            };
+          })
+        knownTreeSitterOrgGrammarRepos);
+
+    in
+      mergeAttrsUnique otherGrammars treeSitterOrgaGrammars;
+
+  # TODO: move to lib
+  mergeAttrsUnique = left: right:
+    let intersect = lib.intersectLists (lib.attrNames left) (lib.attrNames right); in
+    assert
+      lib.assertMsg (intersect == [])
+        (lib.concatStringsSep "\n" [
+          "mergeAttrsUnique: keys in attrset overlapping:"
+          "left: ${lib.generators.toPretty {} (lib.getAttrs intersect left)}"
+          "right: ${lib.generators.toPretty {} (lib.getAttrs intersect right)}"
+        ]);
+    left // right;
+
+
+
   jsonFile = name: val: (formats.json {}).generate name val;
 
   # check the tree-sitter orga repos
@@ -80,18 +119,18 @@ let
   urlEscape = x: x;
 
   # generic bash script to find the latest github release for a repo
-  latestGithubRelease = { owner, repo }: writeShellScript "latest-github-release" ''
+  latestGithubRelease = { orga, repo }: writeShellScript "latest-github-release" ''
     set -euo pipefail
     res=$(${curl}/bin/curl \
       --silent \
-      "https://api.github.com/repos/${urlEscape owner}/${urlEscape repo}/releases/latest")
+      "https://api.github.com/repos/${urlEscape orga}/${urlEscape repo}/releases/latest")
     if [[ "$(printf "%s" "$res" | ${jq}/bin/jq '.message?')" =~ "rate limit" ]]; then
       echo "rate limited" >&2
     fi
     release=$(printf "%s" "$res" | ${jq}/bin/jq '.tag_name')
     # github sometimes returns an empty list even tough there are releases
     if [ "$release" = "null" ]; then
-      echo "uh-oh, latest for ${owner + "/" + repo} is not there, using HEAD" >&2
+      echo "uh-oh, latest for ${orga + "/" + repo} is not there, using HEAD" >&2
       release="HEAD"
     fi
     echo "$release"
@@ -113,18 +152,20 @@ let
   '';
 
   # update one tree-sitter grammar repo and print their nix-prefetch-git output
-  updateGrammar = { owner, repo }: writeShellScript "update-grammar.sh" ''
+  updateGrammar = { orga, repo }: writeShellScript "update-grammar.sh" ''
     set -euo pipefail
-    latest="$(${latestGithubRelease { inherit owner repo; }})"
+    latest="$(${latestGithubRelease { inherit orga repo; }})"
     echo "Fetching latest release ($latest) of ${repo} …" >&2
     ${nix-prefetch-git}/bin/nix-prefetch-git \
       --quiet \
       --no-deepClone \
-      --url "https://github.com/${urlEscape owner}/${urlEscape repo}" \
+      --url "https://github.com/${urlEscape orga}/${urlEscape repo}" \
       --rev "$latest"
     '';
 
-  foreachSh = list: f: lib.concatMapStringsSep "\n" f list;
+  foreachSh = attrs: f:
+    lib.concatMapStringsSep "\n" f
+    (lib.mapAttrsToList (k: v: { name = k; } // v) attrs);
 
   update-all-grammars = writeShellScript "update-all-grammars.sh" ''
     set -euo pipefail
@@ -135,13 +176,13 @@ let
     outputDir="${toString ./.}/grammars"
     echo "writing files to $outputDir" 1>&2
     mkdir -p "$outputDir"
-    ${foreachSh knownTreeSitterOrgGrammarRepos
-      (repo: ''${updateGrammar { owner = "tree-sitter"; inherit repo; }} > $outputDir/${repo}.json'')}
+    ${foreachSh allGrammars
+      ({name, orga, repo}: ''${updateGrammar { inherit orga repo; }} > $outputDir/${name}.json'')}
     ( echo "{"
-      ${foreachSh knownTreeSitterOrgGrammarRepos
-        (repo: ''
+      ${foreachSh allGrammars
+        ({name, ...}: ''
            # indentation hack
-             printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "${repo}" "${repo}"'')}
+             printf "  %s = (builtins.fromJSON (builtins.readFile ./%s.json));\n" "${name}" "${name}"'')}
       echo "}" ) \
       > "$outputDir/default.nix"
   '';

From 0181e103b37abf085dfc127c14a28b86d1077780 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Tue, 29 Dec 2020 13:01:56 +0100
Subject: [PATCH 14/26] tree-sitter: update grammars again with new update
 script

---
 .../parsing/tree-sitter/grammars/default.nix  | 51 ++++++++++---------
 .../tree-sitter/grammars/tree-sitter-lua.json |  8 +--
 .../grammars/tree-sitter-ruby.json            |  8 +--
 3 files changed, 34 insertions(+), 33 deletions(-)

diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
index 18217f69a5b..e1648b84be5 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
@@ -1,30 +1,31 @@
 {
-  tree-sitter-javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
-  tree-sitter-c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
-  tree-sitter-swift = (builtins.fromJSON (builtins.readFile ./tree-sitter-swift.json));
-  tree-sitter-json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
-  tree-sitter-cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
-  tree-sitter-ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
-  tree-sitter-razor = (builtins.fromJSON (builtins.readFile ./tree-sitter-razor.json));
-  tree-sitter-go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
-  tree-sitter-c-sharp = (builtins.fromJSON (builtins.readFile ./tree-sitter-c-sharp.json));
-  tree-sitter-python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
-  tree-sitter-typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
-  tree-sitter-rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
-  tree-sitter-bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
-  tree-sitter-php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
-  tree-sitter-java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
-  tree-sitter-scala = (builtins.fromJSON (builtins.readFile ./tree-sitter-scala.json));
-  tree-sitter-ocaml = (builtins.fromJSON (builtins.readFile ./tree-sitter-ocaml.json));
-  tree-sitter-julia = (builtins.fromJSON (builtins.readFile ./tree-sitter-julia.json));
   tree-sitter-agda = (builtins.fromJSON (builtins.readFile ./tree-sitter-agda.json));
-  tree-sitter-fluent = (builtins.fromJSON (builtins.readFile ./tree-sitter-fluent.json));
-  tree-sitter-html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
-  tree-sitter-haskell = (builtins.fromJSON (builtins.readFile ./tree-sitter-haskell.json));
-  tree-sitter-regex = (builtins.fromJSON (builtins.readFile ./tree-sitter-regex.json));
+  tree-sitter-bash = (builtins.fromJSON (builtins.readFile ./tree-sitter-bash.json));
+  tree-sitter-c = (builtins.fromJSON (builtins.readFile ./tree-sitter-c.json));
+  tree-sitter-c-sharp = (builtins.fromJSON (builtins.readFile ./tree-sitter-c-sharp.json));
+  tree-sitter-cpp = (builtins.fromJSON (builtins.readFile ./tree-sitter-cpp.json));
   tree-sitter-css = (builtins.fromJSON (builtins.readFile ./tree-sitter-css.json));
-  tree-sitter-verilog = (builtins.fromJSON (builtins.readFile ./tree-sitter-verilog.json));
-  tree-sitter-jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
-  tree-sitter-ql = (builtins.fromJSON (builtins.readFile ./tree-sitter-ql.json));
   tree-sitter-embedded-template = (builtins.fromJSON (builtins.readFile ./tree-sitter-embedded-template.json));
+  tree-sitter-fluent = (builtins.fromJSON (builtins.readFile ./tree-sitter-fluent.json));
+  tree-sitter-go = (builtins.fromJSON (builtins.readFile ./tree-sitter-go.json));
+  tree-sitter-haskell = (builtins.fromJSON (builtins.readFile ./tree-sitter-haskell.json));
+  tree-sitter-html = (builtins.fromJSON (builtins.readFile ./tree-sitter-html.json));
+  tree-sitter-java = (builtins.fromJSON (builtins.readFile ./tree-sitter-java.json));
+  tree-sitter-javascript = (builtins.fromJSON (builtins.readFile ./tree-sitter-javascript.json));
+  tree-sitter-jsdoc = (builtins.fromJSON (builtins.readFile ./tree-sitter-jsdoc.json));
+  tree-sitter-json = (builtins.fromJSON (builtins.readFile ./tree-sitter-json.json));
+  tree-sitter-julia = (builtins.fromJSON (builtins.readFile ./tree-sitter-julia.json));
+  tree-sitter-lua = (builtins.fromJSON (builtins.readFile ./tree-sitter-lua.json));
+  tree-sitter-ocaml = (builtins.fromJSON (builtins.readFile ./tree-sitter-ocaml.json));
+  tree-sitter-php = (builtins.fromJSON (builtins.readFile ./tree-sitter-php.json));
+  tree-sitter-python = (builtins.fromJSON (builtins.readFile ./tree-sitter-python.json));
+  tree-sitter-ql = (builtins.fromJSON (builtins.readFile ./tree-sitter-ql.json));
+  tree-sitter-razor = (builtins.fromJSON (builtins.readFile ./tree-sitter-razor.json));
+  tree-sitter-regex = (builtins.fromJSON (builtins.readFile ./tree-sitter-regex.json));
+  tree-sitter-ruby = (builtins.fromJSON (builtins.readFile ./tree-sitter-ruby.json));
+  tree-sitter-rust = (builtins.fromJSON (builtins.readFile ./tree-sitter-rust.json));
+  tree-sitter-scala = (builtins.fromJSON (builtins.readFile ./tree-sitter-scala.json));
+  tree-sitter-swift = (builtins.fromJSON (builtins.readFile ./tree-sitter-swift.json));
+  tree-sitter-typescript = (builtins.fromJSON (builtins.readFile ./tree-sitter-typescript.json));
+  tree-sitter-verilog = (builtins.fromJSON (builtins.readFile ./tree-sitter-verilog.json));
 }
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json
index 267d25a52a0..804d613357e 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-lua.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/nvim-treesitter/tree-sitter-lua",
-  "rev": "278b0ea1f1f3d86afc86faaca0cbbf6c01182a54",
-  "date": "2020-09-08T18:29:23+02:00",
-  "path": "/nix/store/xkaqsw030bf6zd6pivrzrg3d50j1ka1k-tree-sitter-lua",
-  "sha256": "0i8jdp0bbl02h58hfhpild4v3rvy8yk5r5l90navvfxw4ad4f6f3",
+  "rev": "97b757ad3546d68dc1131e6ffb9b1d2a750eea29",
+  "date": "2020-11-25T21:23:57+01:00",
+  "path": "/nix/store/irz0dikk9dahkayi41chcznqq1i3wr84-tree-sitter-lua",
+  "sha256": "1nhg139vk3xm6ip4mhr29z0rprfg4q417z2vlvkz3m9wp8gppzls",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false
diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json
index d5a1767a23c..027478e99f8 100644
--- a/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json
+++ b/pkgs/development/tools/parsing/tree-sitter/grammars/tree-sitter-ruby.json
@@ -1,9 +1,9 @@
 {
   "url": "https://github.com/tree-sitter/tree-sitter-ruby",
-  "rev": "f86d90cce578e28c824afdcdbdca7d031b88c0b0",
-  "date": "2020-11-02T20:49:49-08:00",
-  "path": "/nix/store/gs0hhb1dlgmv0y60zyyz7ki30yvpddfx-tree-sitter-ruby",
-  "sha256": "1n1irdjfmv27p5ia2vqzv8zfsq19pxi0as0wrkb7z7s1lqr1gwrb",
+  "rev": "bb572f60e9538bd11fbde95a54f97522073f1e06",
+  "date": "2020-12-20T12:25:50-08:00",
+  "path": "/nix/store/hkjgvxh8l4drk1z4zkhmd0wcj876x5y1-tree-sitter-ruby",
+  "sha256": "198sfmrb3mxvpfmlwmwfmb3qs075ih0bjmr0ffrdy4nqm4a00cb8",
   "fetchSubmodules": false,
   "deepClone": false,
   "leaveDotGit": false

From 80bff15d7b9bfc18d2734607654ce7525f8c1097 Mon Sep 17 00:00:00 2001
From: freezeboy <freezeboy@users.noreply.github.com>
Date: Wed, 15 Jul 2020 09:43:19 +0200
Subject: [PATCH 15/26] colobot: init at 0.1.12-alpha

Initial integration without the game musics to reduce the size of the
game. It works without them

Data are in another derivation to let them live their live independently
as they are quite big.

Limited to linux platform only because physfs is currently broken on
darwin.
---
 pkgs/games/colobot/data.nix     | 41 +++++++++++++++++++++++++++++++++
 pkgs/games/colobot/default.nix  | 41 +++++++++++++++++++++++++++++++++
 pkgs/top-level/all-packages.nix |  2 ++
 3 files changed, 84 insertions(+)
 create mode 100644 pkgs/games/colobot/data.nix
 create mode 100644 pkgs/games/colobot/default.nix

diff --git a/pkgs/games/colobot/data.nix b/pkgs/games/colobot/data.nix
new file mode 100644
index 00000000000..f1a4782fad6
--- /dev/null
+++ b/pkgs/games/colobot/data.nix
@@ -0,0 +1,41 @@
+{ stdenv, lib, fetchFromGitHub, cmake
+, gettext, vorbis-tools
+, xmlstarlet, doxygen, python3 }:
+
+stdenv.mkDerivation rec {
+  pname = "colobot-data";
+  version = "0.1.12-alpha";
+
+  src = fetchFromGitHub {
+    owner = "colobot";
+    repo = "colobot-data";
+    rev = "colobot-gold-${version}";
+    sha256 = "1vm33s52ymwd03x24i9bqiglw5v3wgd7rlzyx9r5ww0nnqzwbwi6";
+  };
+
+  nativeBuildInputs = [ cmake vorbis-tools xmlstarlet doxygen python3 ];
+  buildInputs = [ gettext ];
+
+  enableParallelBuilding = false;
+  # Build procedure requires the data folder
+  patchPhase = ''
+    cp -r $src localSrc
+    chmod +w localSrc/help/bots/po
+    find -type d -exec chmod +w {} \;
+    for po in localSrc/help/{bots,cbot,object,generic,programs}/po/* localSrc/levels/*{/*/*,}/po/*; do
+      rm $po
+      touch $po
+    done
+    # skip music
+    rm localSrc/music/CMakeLists.txt
+    cd localSrc
+  '';
+
+  meta = with stdenv.lib; {
+    homepage = "https://colobot.info/";
+    description = "Game data for colobot";
+    license = licenses.gpl3;
+    maintainers = with maintainers; [ freezeboy ];
+    platforms = platforms.linux;
+  };
+}
diff --git a/pkgs/games/colobot/default.nix b/pkgs/games/colobot/default.nix
new file mode 100644
index 00000000000..9c0cbe7949c
--- /dev/null
+++ b/pkgs/games/colobot/default.nix
@@ -0,0 +1,41 @@
+{ stdenv, fetchFromGitHub, cmake, boost, SDL2, SDL2_image, SDL2_ttf, libpng
+, glew, gettext, libsndfile, libvorbis, libogg, physfs, openal
+, xmlstarlet, doxygen, python3, callPackage }:
+
+let
+  colobot-data = callPackage ./data.nix {};
+in
+stdenv.mkDerivation rec {
+  pname = "colobot";
+  # Maybe require an update to package colobot-data as well
+  # in file data.nix next to this one
+  version = "0.1.12-alpha";
+
+  src = fetchFromGitHub {
+    owner = "colobot";
+    repo = "colobot";
+    rev = "colobot-gold-${version}";
+    sha256 = "0viq5s4zqs33an7rdmc3anf74ml7mwwcwf60alhvp9hj5jr547s2";
+  };
+
+  nativeBuildInputs = [ cmake xmlstarlet doxygen python3 ];
+  buildInputs = [ boost SDL2 SDL2_image SDL2_ttf libpng glew gettext libsndfile libvorbis libogg physfs openal ];
+
+  enableParallelBuilding = false;
+
+  # The binary ends in games directoy
+  postInstall = ''
+    mv $out/games $out/bin
+    for contents in ${colobot-data}/share/games/colobot/*; do
+      ln -s $contents $out/share/games/colobot
+    done
+  '';
+
+  meta = with stdenv.lib; {
+    homepage = "https://colobot.info/";
+    description = "Colobot: Gold Edition is a real-time strategy game, where you can program your bots";
+    license = licenses.gpl3;
+    maintainers = with maintainers; [ freezeboy ];
+    platforms = platforms.linux;
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 930189a63ba..ed2c8dd613e 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -192,6 +192,8 @@ in
 
   corgi = callPackage ../development/tools/corgi { };
 
+  colobot = callPackage ../games/colobot {};
+
   colorz = callPackage ../tools/misc/colorz { };
 
   colorpicker = callPackage ../tools/misc/colorpicker { };

From 00359f25e2e528530c70cecf302186e0e49cc1d4 Mon Sep 17 00:00:00 2001
From: Profpatsch <mail@profpatsch.de>
Date: Tue, 29 Dec 2020 13:34:32 +0100
Subject: [PATCH 16/26] tree-sitter: test building of grammars

---
 pkgs/development/tools/parsing/tree-sitter/default.nix | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix
index a70757ca15e..72150b5ac0b 100644
--- a/pkgs/development/tools/parsing/tree-sitter/default.nix
+++ b/pkgs/development/tools/parsing/tree-sitter/default.nix
@@ -101,6 +101,11 @@ in rustPlatform.buildRustPackage {
     };
     inherit grammars;
     inherit builtGrammars;
+
+    tests = {
+      # make sure all grammars build
+      builtGrammars = lib.recurseIntoAttrs builtGrammars;
+    };
   };
 
   meta = {

From 7e90346e1605cf24ac26f9a6d0cdfae1f1f667eb Mon Sep 17 00:00:00 2001
From: Symphorien Gibol <symphorien+git@xlumurb.eu>
Date: Mon, 6 Jul 2020 12:00:00 +0000
Subject: [PATCH 17/26] kcollectd: init at 0.11.99.0

Co-authored-by: davidak <davidak@users.noreply.github.com>
---
 pkgs/tools/misc/kcollectd/default.nix | 56 +++++++++++++++++++++++++++
 pkgs/top-level/all-packages.nix       |  2 +
 2 files changed, 58 insertions(+)
 create mode 100644 pkgs/tools/misc/kcollectd/default.nix

diff --git a/pkgs/tools/misc/kcollectd/default.nix b/pkgs/tools/misc/kcollectd/default.nix
new file mode 100644
index 00000000000..ffc64b1455a
--- /dev/null
+++ b/pkgs/tools/misc/kcollectd/default.nix
@@ -0,0 +1,56 @@
+{ lib
+, fetchFromGitLab
+, mkDerivation
+, qtbase
+, cmake
+, kconfig
+, kio
+, kiconthemes
+, kxmlgui
+, ki18n
+, kguiaddons
+, extra-cmake-modules
+, boost
+, shared-mime-info
+, rrdtool
+, breeze-icons
+}:
+
+mkDerivation rec {
+  pname = "kcollectd";
+  version = "0.11.99.0";
+  src = fetchFromGitLab {
+    owner = "aerusso";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "0h4ymvzihzbmyv3z0bp28g94wxc6c7lgi3my0xbka3advxr811gn";
+  };
+
+  nativeBuildInputs = [
+    cmake
+    extra-cmake-modules
+    shared-mime-info
+  ];
+
+  buildInputs = [
+    qtbase
+    kconfig
+    kio
+    kxmlgui
+    kiconthemes
+    ki18n
+    kguiaddons
+    boost
+    rrdtool
+    # otherwise some buttons are blank
+    breeze-icons
+  ];
+
+  meta = with lib; {
+    description = "A graphical frontend to collectd";
+    homepage = "https://www.antonioerusso.com/projects/kcollectd/";
+    maintainers = [ maintainers.symphorien ];
+    license = [ lib.licenses.gpl3Plus ];
+    platforms = lib.platforms.linux;
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 1e3f0eaf06a..9c27c778f2e 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -5301,6 +5301,8 @@ in
 
   file-rename = callPackage ../tools/filesystems/file-rename { };
 
+  kcollectd = libsForQt5.callPackage ../tools/misc/kcollectd {};
+
   kea = callPackage ../tools/networking/kea { };
 
   keysmith = libsForQt5.callPackage ../tools/security/keysmith { };

From dcab5b7d573210608728b65c314c482eac35b935 Mon Sep 17 00:00:00 2001
From: SCOTT-HAMILTON <sgn.hamilton+github@protonmail.com>
Date: Tue, 29 Dec 2020 17:40:22 +0100
Subject: [PATCH 18/26] ksmoothdock: qt514 -> qt515 (fix build)

---
 pkgs/applications/misc/ksmoothdock/default.nix | 5 +++++
 pkgs/top-level/all-packages.nix                | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/pkgs/applications/misc/ksmoothdock/default.nix b/pkgs/applications/misc/ksmoothdock/default.nix
index 9538329bf9b..e5cb6f5816e 100644
--- a/pkgs/applications/misc/ksmoothdock/default.nix
+++ b/pkgs/applications/misc/ksmoothdock/default.nix
@@ -25,6 +25,11 @@ mkDerivation rec {
       url = "https://github.com/dangvd/ksmoothdock/commit/00799bef8a1c1fe61ef9274866267d9fe9194041.patch";
       sha256 = "1nmb7gf1ggzicxz8k4fd67xhwjy404myqzjpgjym66wqxm0arni4";
     })
+    # Pull request to fix build on Qt 5.15 https://github.com/dangvd/ksmoothdock/pull/123
+    (fetchpatch {
+      url = "https://github.com/dangvd/ksmoothdock/commit/259527aacadb0fd9110d4425b9bf41a15bedce72.patch";
+      sha256 = "12nj58v9qqrynarn3gpywih3w27mr4n51z1b8mh0rfbnd2kib8dc";
+    })
   ];
 
   nativeBuildInputs = [ cmake extra-cmake-modules ];
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 26eb622ab9d..7e40b0a7541 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -5124,7 +5124,7 @@ in
 
   peruse = libsForQt5.callPackage ../tools/misc/peruse { };
 
-  ksmoothdock = libsForQt514.callPackage ../applications/misc/ksmoothdock { };
+  ksmoothdock = libsForQt5.callPackage ../applications/misc/ksmoothdock { };
 
   kstars = libsForQt514.callPackage ../applications/science/astronomy/kstars { };
 

From 77c70abb813783ff71af5c362e1c028f2f870d5d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sandro=20J=C3=A4ckel?= <sandro.jaeckel@gmail.com>
Date: Wed, 30 Dec 2020 00:04:35 +0100
Subject: [PATCH 19/26] torchvision: 0.2.1 -> 0.8.2

---
 .../python-modules/torchvision/default.nix    | 52 +++++++++++++------
 1 file changed, 35 insertions(+), 17 deletions(-)

diff --git a/pkgs/development/python-modules/torchvision/default.nix b/pkgs/development/python-modules/torchvision/default.nix
index 4d371579099..12af086a07d 100644
--- a/pkgs/development/python-modules/torchvision/default.nix
+++ b/pkgs/development/python-modules/torchvision/default.nix
@@ -1,30 +1,48 @@
-{ buildPythonPackage
-, fetchPypi
-, six
+{ stdenv
+, buildPythonPackage
+, fetchFromGitHub
+, ninja
+, which
+, libjpeg_turbo
+, libpng
 , numpy
+, scipy
 , pillow
 , pytorch
-, lib
+, pytestCheckHook
 }:
 
 buildPythonPackage rec {
-  version = "0.2.1";
-  pname   = "torchvision";
+  pname = "torchvision";
+  version = "0.8.2";
 
-  format = "wheel";
-
-  src = fetchPypi {
-    inherit pname version;
-    format = "wheel";
-    sha256 = "18gvdabkmzfjg47ns0lw38mf85ry28nq1mas5rzlwvb4l5zmw2ms";
+  src = fetchFromGitHub {
+    owner = "pytorch";
+    repo = "vision";
+    rev = "v${version}";
+    sha256 = "0yhpbq7linrk2qp5jxsvlgkmwa5bn38s9kcswy4jzvmx1fjbkpq0";
   };
 
-  propagatedBuildInputs = [ six numpy pillow pytorch ];
+  nativeBuildInputs = [ libpng ninja which ];
 
-  meta = {
+  TORCHVISION_INCLUDE="${libjpeg_turbo.dev}/include/";
+  TORCHVISION_LIBRARY="${libjpeg_turbo}/lib/";
+
+  buildInputs = [ libjpeg_turbo libpng ];
+
+  propagatedBuildInputs = [ numpy pillow pytorch scipy ];
+
+  # checks fail to import _C.so and various other failures
+  doCheck = false;
+
+  checkInputs = [ pytestCheckHook ];
+
+  pytestFlagsArray = [ "--ignore=test/test_datasets_download.py" ];
+
+  meta = with stdenv.lib; {
     description = "PyTorch vision library";
-    homepage    = "https://pytorch.org/";
-    license     = lib.licenses.bsd3;
-    maintainers = with lib.maintainers; [ ericsagnes ];
+    homepage = "https://pytorch.org/";
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ ericsagnes SuperSandro2000 ];
   };
 }

From f91d7cf059f4eb7825214febba7b4ddf003e34ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Wed, 30 Dec 2020 06:58:57 +0100
Subject: [PATCH 20/26] cntr: 1.2.1 -> 1.4.1

---
 pkgs/applications/virtualization/cntr/default.nix | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/pkgs/applications/virtualization/cntr/default.nix b/pkgs/applications/virtualization/cntr/default.nix
index e3ab1c6c2f8..ab9b63eb79b 100644
--- a/pkgs/applications/virtualization/cntr/default.nix
+++ b/pkgs/applications/virtualization/cntr/default.nix
@@ -2,23 +2,22 @@
 
 rustPlatform.buildRustPackage rec {
   pname = "cntr";
-  version = "1.2.1";
+  version = "1.4.1";
 
   src = fetchFromGitHub {
     owner = "Mic92";
     repo = "cntr";
     rev = version;
-    sha256 = "0dhfz7aj3cqi974ybf0axchih40rzrs9m8bxhwz1hgig57aisfc0";
+    sha256 = "sha256-4ogyOKuz6702/sOQNvE+UP+cvQrPPU3VjL4b0FUfRNw=";
   };
 
-  cargoSha256 = "088drkpkgq8psv5j6igxyhfvvbalzg6nd98r9z0nxkawck5i2clz";
+  cargoSha256 = "sha256-lblvun2T1qpFiowld77Ti2MFPzhs5pOWWRbErORXYCM=";
 
   meta = with stdenv.lib; {
     description = "A container debugging tool based on FUSE";
     homepage = "https://github.com/Mic92/cntr";
     license = licenses.mit;
-    # aarch64 support will be fixed soon
-    platforms = [ "x86_64-linux" ];
+    platforms = platforms.linux;
     maintainers = [ maintainers.mic92 ];
   };
 }

From d198cbabe7c6c78a1b161ee139e1debd6e5d6e92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= <joerg@thalheim.io>
Date: Wed, 30 Dec 2020 07:42:16 +0100
Subject: [PATCH 21/26] torchvision: update reason why tests fails

---
 .../python-modules/torchvision/default.nix         | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/pkgs/development/python-modules/torchvision/default.nix b/pkgs/development/python-modules/torchvision/default.nix
index 12af086a07d..ea702a78fe1 100644
--- a/pkgs/development/python-modules/torchvision/default.nix
+++ b/pkgs/development/python-modules/torchvision/default.nix
@@ -9,7 +9,7 @@
 , scipy
 , pillow
 , pytorch
-, pytestCheckHook
+, pytest
 }:
 
 buildPythonPackage rec {
@@ -25,19 +25,21 @@ buildPythonPackage rec {
 
   nativeBuildInputs = [ libpng ninja which ];
 
-  TORCHVISION_INCLUDE="${libjpeg_turbo.dev}/include/";
-  TORCHVISION_LIBRARY="${libjpeg_turbo}/lib/";
+  TORCHVISION_INCLUDE = "${libjpeg_turbo.dev}/include/";
+  TORCHVISION_LIBRARY = "${libjpeg_turbo}/lib/";
 
   buildInputs = [ libjpeg_turbo libpng ];
 
   propagatedBuildInputs = [ numpy pillow pytorch scipy ];
 
-  # checks fail to import _C.so and various other failures
+  # tries to download many datasets for tests
   doCheck = false;
 
-  checkInputs = [ pytestCheckHook ];
+  checkPhase = ''
+    HOME=$TMPDIR py.test test --ignore=test/test_datasets_download.py
+  '';
 
-  pytestFlagsArray = [ "--ignore=test/test_datasets_download.py" ];
+  checkInputs = [ pytest ];
 
   meta = with stdenv.lib; {
     description = "PyTorch vision library";

From 1dc2b42dfe426445bddc45143e639eaa1be8252f Mon Sep 17 00:00:00 2001
From: Fabian Affolter <mail@fabian-affolter.ch>
Date: Wed, 30 Dec 2020 10:32:41 +0100
Subject: [PATCH 22/26] python3Packages.adb-shell: 0.2.3 -> 0.3.0

---
 pkgs/development/python-modules/adb-shell/default.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkgs/development/python-modules/adb-shell/default.nix b/pkgs/development/python-modules/adb-shell/default.nix
index 9b2de53201b..9cd3fe36233 100644
--- a/pkgs/development/python-modules/adb-shell/default.nix
+++ b/pkgs/development/python-modules/adb-shell/default.nix
@@ -3,7 +3,7 @@
 
 buildPythonPackage rec {
   pname = "adb-shell";
-  version = "0.2.3";
+  version = "0.3.0";
 
   disabled = !isPy3k;
 
@@ -12,7 +12,7 @@ buildPythonPackage rec {
     owner = "JeffLIrion";
     repo = "adb_shell";
     rev = "v${version}";
-    sha256 = "1ay598avmg656cxnc9phdx43z1plsrfjf9png9jwjwyhyjjiqxil";
+    sha256 = "0qnlhcd58zxh39cd5xzdx8yc5hc0pf8kix4rbn4avsapwb0l75n2";
   };
 
   propagatedBuildInputs = [ aiofiles cryptography libusb1 pyasn1 rsa ];

From 06e80a5930a2498e0903a577a7366cf7f4b5e034 Mon Sep 17 00:00:00 2001
From: wishfort36 <42300264+wishfort36@users.noreply.github.com>
Date: Tue, 29 Dec 2020 22:13:50 +0100
Subject: [PATCH 23/26] tiramisu: init at 1.0

---
 pkgs/applications/misc/tiramisu/default.nix | 36 +++++++++++++++++++++
 pkgs/top-level/all-packages.nix             |  2 ++
 2 files changed, 38 insertions(+)
 create mode 100644 pkgs/applications/misc/tiramisu/default.nix

diff --git a/pkgs/applications/misc/tiramisu/default.nix b/pkgs/applications/misc/tiramisu/default.nix
new file mode 100644
index 00000000000..e5cc9a1c178
--- /dev/null
+++ b/pkgs/applications/misc/tiramisu/default.nix
@@ -0,0 +1,36 @@
+{ stdenv, fetchFromGitHub, pkg-config, glib }:
+
+stdenv.mkDerivation rec {
+  pname = "tiramisu";
+  version = "1.0";
+
+  src = fetchFromGitHub {
+    owner = "Sweets";
+    repo = pname;
+    rev = version;
+    sha256 = "0aw17riwgrhsmcndzh7sw2zw8xvn3d203c2gcrqi9nk5pa7fwp9m";
+  };
+
+  postPatch = ''
+    sed -i 's/printf(element_delimiter)/printf("%s", element_delimiter)/' src/output.c
+  '';
+
+  buildInputs = [ glib ];
+
+  nativeBuildInputs = [ pkg-config ];
+
+  makeFlags = [ "PREFIX=$(out)" ];
+
+  meta = with stdenv.lib; {
+    description = "Desktop notifications, the UNIX way";
+    longDescription = ''
+    tiramisu is a notification daemon based on dunst that outputs notifications
+    to STDOUT in order to allow the user to process notifications any way they
+    prefer.
+    '';
+    homepage = "https://github.com/Sweets/tiramisu";
+    license = licenses.mit;
+    platforms = platforms.linux;
+    maintainers = with maintainers; [ wishfort36 ];
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 27c565bce0c..44e3be6eb19 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -22080,6 +22080,8 @@ in
 
   swaylock-effects = callPackage ../applications/window-managers/sway/lock-effects.nix { };
 
+  tiramisu = callPackage ../applications/misc/tiramisu { };
+
   waybar = callPackage ../applications/misc/waybar {};
 
   hikari = callPackage ../applications/window-managers/hikari { };

From 027c952090e5fda9badeff944b6b81ca25168f6e Mon Sep 17 00:00:00 2001
From: Frederik Rietdijk <fridh@fridh.nl>
Date: Wed, 30 Dec 2020 10:46:35 +0100
Subject: [PATCH 24/26] Revert "libjpeg_turbo: apply cmake patch for darwin"

cmake is now on 3.19.2 solving the bug that required this patch.

This reverts commit 5524c43f2c0cc246de33d6eff378dcad843b2617.
---
 pkgs/top-level/all-packages.nix | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 29320170bcb..c92c54c21fe 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -14509,18 +14509,7 @@ in
 
   libjpeg_original = callPackage ../development/libraries/libjpeg { };
   # also known as libturbojpeg
-  libjpeg_turbo = callPackage ../development/libraries/libjpeg-turbo (lib.optionalAttrs stdenv.isDarwin {
-    # cmake 3.19.1 has a bug. So far only noticed with this package.
-    # https://github.com/NixOS/nixpkgs/issues/105854
-    cmake = cmake.overrideAttrs(oldAttrs: {
-      patches = oldAttrs.patches ++ [
-        (fetchpatch {
-          url = "https://gitlab.kitware.com/cmake/cmake/-/commit/fcabf4a47e0c441ff80fad8f34e388b16738bd33.patch";
-          sha256 = "bVrjY8omtAEKe8G76hGtmO54LKJvhx3RTW6OF6Y7rsU=";
-        })
-      ];
-    });
-  });
+  libjpeg_turbo = callPackage ../development/libraries/libjpeg-turbo { };
   libjpeg = libjpeg_turbo;
 
   libjreen = callPackage ../development/libraries/libjreen { };

From 95f685da60224a20df45361f3b92a41a69887eb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= <v@cunat.cz>
Date: Wed, 30 Dec 2020 09:04:15 +0100
Subject: [PATCH 25/26] python3Packages.mypy: don't use mypyc on 32-bit
 platforms

Almost all i686-linux tests got blocked because of this problem:
  https://hydra.nixos.org/eval/1638038#tabs-now-fail
It regressed in PR #105462 (commit ad26cb9ee8).

Now I tested that at least some test got fixed:
  nix build -f nixos/release-combined.nix nixos.tests.knot.i686-linux
This change won't even cause any rebuild on 64-bit platforms,
and using nix booleans seems nicer anyway.
---
 pkgs/development/python-modules/mypy/default.nix | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pkgs/development/python-modules/mypy/default.nix b/pkgs/development/python-modules/mypy/default.nix
index 85ceaf304c2..1b8d763eb6f 100644
--- a/pkgs/development/python-modules/mypy/default.nix
+++ b/pkgs/development/python-modules/mypy/default.nix
@@ -53,7 +53,8 @@ buildPythonPackage rec {
 
   # Compile mypy with mypyc, which makes mypy about 4 times faster. The compiled
   # version is also the default in the wheels on Pypi that include binaries.
-  MYPY_USE_MYPYC = "1";
+  # is64bit: unfortunately the build would exhaust all possible memory on i686-linux.
+  MYPY_USE_MYPYC = stdenv.buildPlatform.is64bit;
 
   meta = with stdenv.lib; {
     description = "Optional static typing for Python";

From e8dd1fa1d9b3332f790202facdb6b37aee492653 Mon Sep 17 00:00:00 2001
From: SebastienWae <55792872+SebastienWae@users.noreply.github.com>
Date: Wed, 30 Dec 2020 11:58:49 +0100
Subject: [PATCH 26/26] kernelPackages.broadcom_sta: fix build for 5.9
 (#106760)

* broadcom: fix build 5.9

Patch to fix the build error for the 5.9 Linux kernel.

* broadcom: swith 5.9 patch to Joan Bruguera's version
Switch the current patch for 5.9 to Joan Bruguera's version which is cleaner and also works for 5.10
---
 .../linux/broadcom-sta/default.nix            |   2 +
 .../linux/broadcom-sta/linux-5.9.patch        | 184 ++++++++++++++++++
 2 files changed, 186 insertions(+)
 create mode 100644 pkgs/os-specific/linux/broadcom-sta/linux-5.9.patch

diff --git a/pkgs/os-specific/linux/broadcom-sta/default.nix b/pkgs/os-specific/linux/broadcom-sta/default.nix
index ecaa3896044..8e2271b9f7f 100644
--- a/pkgs/os-specific/linux/broadcom-sta/default.nix
+++ b/pkgs/os-specific/linux/broadcom-sta/default.nix
@@ -37,6 +37,8 @@ stdenv.mkDerivation {
     ./linux-5.1.patch
     # source: https://salsa.debian.org/Herrie82-guest/broadcom-sta/-/commit/247307926e5540ad574a17c062c8da76990d056f
     ./linux-5.6.patch
+    # source: https://gist.github.com/joanbm/5c640ac074d27fd1d82c74a5b67a1290
+    ./linux-5.9.patch
     ./null-pointer-fix.patch
     ./gcc.patch
   ];
diff --git a/pkgs/os-specific/linux/broadcom-sta/linux-5.9.patch b/pkgs/os-specific/linux/broadcom-sta/linux-5.9.patch
new file mode 100644
index 00000000000..2a4e6fa89cc
--- /dev/null
+++ b/pkgs/os-specific/linux/broadcom-sta/linux-5.9.patch
@@ -0,0 +1,184 @@
+diff --git a/src/wl/sys/wl_cfg80211_hybrid.c b/src/wl/sys/wl_cfg80211_hybrid.c
+index 4b3298f..c45ad48 100644
+--- a/src/wl/sys/wl_cfg80211_hybrid.c
++++ b/src/wl/sys/wl_cfg80211_hybrid.c
+@@ -41,6 +41,7 @@
+ #include <wlioctl.h>
+ #include <proto/802.11.h>
+ #include <wl_cfg80211_hybrid.h>
++#include <wl_linux.h>
+ 
+ #define EVENT_TYPE(e) dtoh32((e)->event_type)
+ #define EVENT_FLAGS(e) dtoh16((e)->flags)
+@@ -442,30 +443,7 @@ static void key_endian_to_host(struct wl_wsec_key *key)
+ static s32
+ wl_dev_ioctl(struct net_device *dev, u32 cmd, void *arg, u32 len)
+ {
+-	struct ifreq ifr;
+-	struct wl_ioctl ioc;
+-	mm_segment_t fs;
+-	s32 err = 0;
+-
+-	BUG_ON(len < sizeof(int));
+-
+-	memset(&ioc, 0, sizeof(ioc));
+-	ioc.cmd = cmd;
+-	ioc.buf = arg;
+-	ioc.len = len;
+-	strcpy(ifr.ifr_name, dev->name);
+-	ifr.ifr_data = (caddr_t)&ioc;
+-
+-	fs = get_fs();
+-	set_fs(KERNEL_DS);
+-#if defined(WL_USE_NETDEV_OPS)
+-	err = dev->netdev_ops->ndo_do_ioctl(dev, &ifr, SIOCDEVPRIVATE);
+-#else
+-	err = dev->do_ioctl(dev, &ifr, SIOCDEVPRIVATE);
+-#endif
+-	set_fs(fs);
+-
+-	return err;
++	return wlc_ioctl_internal(dev, cmd, arg, len);
+ }
+ 
+ static s32
+diff --git a/src/wl/sys/wl_iw.c b/src/wl/sys/wl_iw.c
+index 9c3c74e..e346b15 100644
+--- a/src/wl/sys/wl_iw.c
++++ b/src/wl/sys/wl_iw.c
+@@ -37,6 +37,7 @@ typedef const struct si_pub	si_t;
+ 
+ #include <wl_dbg.h>
+ #include <wl_iw.h>
++#include <wl_linux.h>
+ 
+ extern bool wl_iw_conn_status_str(uint32 event_type, uint32 status,
+ 	uint32 reason, char* stringBuf, uint buflen);
+@@ -103,29 +104,7 @@ dev_wlc_ioctl(
+ 	int len
+ )
+ {
+-	struct ifreq ifr;
+-	wl_ioctl_t ioc;
+-	mm_segment_t fs;
+-	int ret;
+-
+-	memset(&ioc, 0, sizeof(ioc));
+-	ioc.cmd = cmd;
+-	ioc.buf = arg;
+-	ioc.len = len;
+-
+-	strcpy(ifr.ifr_name, dev->name);
+-	ifr.ifr_data = (caddr_t) &ioc;
+-
+-	fs = get_fs();
+-	set_fs(KERNEL_DS);
+-#if defined(WL_USE_NETDEV_OPS)
+-	ret = dev->netdev_ops->ndo_do_ioctl(dev, &ifr, SIOCDEVPRIVATE);
+-#else
+-	ret = dev->do_ioctl(dev, &ifr, SIOCDEVPRIVATE);
+-#endif
+-	set_fs(fs);
+-
+-	return ret;
++	return wlc_ioctl_internal(dev, cmd, arg, len);
+ }
+ 
+ static int
+diff --git a/src/wl/sys/wl_linux.c b/src/wl/sys/wl_linux.c
+index c990c70..5bb9480 100644
+--- a/src/wl/sys/wl_linux.c
++++ b/src/wl/sys/wl_linux.c
+@@ -1664,10 +1664,7 @@ wl_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
+ 		goto done2;
+ 	}
+ 
+-	if (segment_eq(get_fs(), KERNEL_DS))
+-		buf = ioc.buf;
+-
+-	else if (ioc.buf) {
++	if (ioc.buf) {
+ 		if (!(buf = (void *) MALLOC(wl->osh, MAX(ioc.len, WLC_IOCTL_MAXLEN)))) {
+ 			bcmerror = BCME_NORESOURCE;
+ 			goto done2;
+@@ -1688,7 +1685,7 @@ wl_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
+ 	WL_UNLOCK(wl);
+ 
+ done1:
+-	if (ioc.buf && (ioc.buf != buf)) {
++	if (ioc.buf) {
+ 		if (copy_to_user(ioc.buf, buf, ioc.len))
+ 			bcmerror = BCME_BADADDR;
+ 		MFREE(wl->osh, buf, MAX(ioc.len, WLC_IOCTL_MAXLEN));
+@@ -1701,6 +1698,39 @@ done2:
+ 	return (OSL_ERROR(bcmerror));
+ }
+ 
++int
++wlc_ioctl_internal(struct net_device *dev, int cmd, void *buf, int len)
++{
++	wl_info_t *wl;
++	wl_if_t *wlif;
++	int bcmerror;
++
++	if (!dev)
++		return -ENETDOWN;
++
++	wl = WL_INFO(dev);
++	wlif = WL_DEV_IF(dev);
++	if (wlif == NULL || wl == NULL || wl->dev == NULL)
++		return -ENETDOWN;
++
++	bcmerror = 0;
++
++	WL_TRACE(("wl%d: wlc_ioctl_internal: cmd 0x%x\n", wl->pub->unit, cmd));
++
++	WL_LOCK(wl);
++	if (!capable(CAP_NET_ADMIN)) {
++		bcmerror = BCME_EPERM;
++	} else {
++		bcmerror = wlc_ioctl(wl->wlc, cmd, buf, len, wlif->wlcif);
++	}
++	WL_UNLOCK(wl);
++
++	ASSERT(VALID_BCMERROR(bcmerror));
++	if (bcmerror != 0)
++		wl->pub->bcmerror = bcmerror;
++	return (OSL_ERROR(bcmerror));
++}
++
+ static struct net_device_stats*
+ wl_get_stats(struct net_device *dev)
+ {
+diff --git a/src/wl/sys/wl_linux.h b/src/wl/sys/wl_linux.h
+index 5b1048e..c8c1f41 100644
+--- a/src/wl/sys/wl_linux.h
++++ b/src/wl/sys/wl_linux.h
+@@ -22,6 +22,7 @@
+ #define _wl_linux_h_
+ 
+ #include <wlc_types.h>
++#include <wlc_pub.h>
+ 
+ typedef struct wl_timer {
+ 	struct timer_list 	timer;
+@@ -187,6 +188,7 @@ extern irqreturn_t wl_isr(int irq, void *dev_id, struct pt_regs *ptregs);
+ extern int __devinit wl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
+ extern void wl_free(wl_info_t *wl);
+ extern int  wl_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd);
++extern int wlc_ioctl_internal(struct net_device *dev, int cmd, void *buf, int len);
+ extern struct net_device * wl_netdev_get(wl_info_t *wl);
+ 
+ #endif 
+diff --git a/src/wl/sys/wlc_pub.h b/src/wl/sys/wlc_pub.h
+index 53a98b8..2b5a029 100644
+--- a/src/wl/sys/wlc_pub.h
++++ b/src/wl/sys/wlc_pub.h
+@@ -24,6 +24,7 @@
+ 
+ #include <wlc_types.h>
+ #include <wlc_utils.h>
++#include <siutils.h>
+ #include "proto/802.11.h"
+ #include "proto/bcmevent.h"
+