diff --git a/pkgs/development/python-modules/matplotlib/default.nix b/pkgs/development/python-modules/matplotlib/default.nix
index 45f31739746..732a6c89f43 100644
--- a/pkgs/development/python-modules/matplotlib/default.nix
+++ b/pkgs/development/python-modules/matplotlib/default.nix
@@ -45,7 +45,7 @@ buildPythonPackage rec {
     ++ stdenv.lib.optionals enableTk [ tcl tk tkinter libX11 ]
     ++ stdenv.lib.optionals enableQt [ pyqt5 ];
 
-  setup_cfg = ./setup.cfg;
+  setup_cfg = if stdenv.isDarwin then ./setup-darwin.cfg else ./setup.cfg;
   preBuild = ''
     cp "$setup_cfg" ./setup.cfg
   '';
diff --git a/pkgs/development/python-modules/matplotlib/setup-darwin.cfg b/pkgs/development/python-modules/matplotlib/setup-darwin.cfg
new file mode 100644
index 00000000000..f6463d9c574
--- /dev/null
+++ b/pkgs/development/python-modules/matplotlib/setup-darwin.cfg
@@ -0,0 +1,7 @@
+[directories]
+basedirlist = .
+
+[libs]
+system_freetype = true
+# LTO not working in darwin stdenv, see #19312
+enable_lto = false