# HG changeset patch
# User Malcolm <malcolm@terrateam.io>
# Date 1739054380 -3600
#      Sat Feb 08 23:39:40 2025 +0100
# Node ID be41ee42c393330b3d31f3e31bb52f1d85ee096a
# Parent  5438c7ef4839ca848ceac0172ea4befa4612314d
REFACTOR Improve performance by 10x by batching SQL query

diff --git a/README.md b/README.md
--- a/README.md
+++ b/README.md
@@ -2,61 +2,62 @@
 
 # Table of Contents
 
-1.  [About](#org705abbd)
-    1.  [Why Does This Exist?](#org1e147e4)
-2.  [Parallelism](#org95ace50)
-3.  [Content Hash Based Builds](#orga0bd8ef)
-4.  [Terminology](#org14f2d42)
-5.  [Projects](#orge184885)
-    1.  [Ocaml](#org599210b)
-    2.  [Third-party](#org5025403)
-6.  [Dependencies](#orgba472af)
-7.  [Configuration](#orgfbc6674)
-    1.  [Configuration Variables For Projects](#orge2f869d)
-    2.  [Configuration Variables For Tests](#orgd259166)
-    3.  [Outputting the configuration](#org382a124)
-8.  [Produced Files](#orge707b46)
-9.  [Documentation](#org9d84515)
-10. [Selectors](#orge5261a0)
-11. [Changelog](#org96c1e30)
-    1.  [1.0.0](#org9783e67)
-    2.  [1.1.0](#orgc6cbf55)
-    3.  [2.0.0](#org73158b8)
-    4.  [3.0.0](#org4f7c40c)
-    5.  [3.0.1](#org67c028d)
-    6.  [3.0.2](#orgf7b00c6)
-    7.  [3.0.3](#org54e2aa9)
-    8.  [3.0.4](#orge538f05)
-    9.  [3.1.0](#org36f73d1)
-    10. [3.1.1](#org517f089)
-    11. [4.11](#org5b01e9c)
-    12. [4.12](#orgb36a098)
-    13. [4.13](#orgefc38b9)
-    14. [5.14](#orgd9d3d19)
-    15. [5.15](#org04bafd2)
-    16. [5.16](#org8c5d0f6)
-    17. [5.17 (bug release)](#org1757356)
-    18. [5.18](#org8ccbcfb)
-    19. [5.19](#org7a636a0)
-    20. [6.20](#org252971c)
-    21. [5.21](#org51426d2)
-    22. [5.22](#orgdb47994)
-    23. [5.23](#orge28c50d)
-    24. [5.24](#org60c1f2c)
-    25. [5.25](#orgeb1d233)
-    26. [5.26 (bug release)](#org1d9ff84)
-    27. [5.27](#org8a0c77a)
-    28. [5.28](#orgc9518f4)
-    29. [5.29](#org01640d7)
-    30. [6.43](#orgc817f12)
-    31. [6.44](#orgcd3d50a)
-    32. [6.45](#org8731d09)
-    33. [6.46](#orge7cf45c)
-    34. [6.47](#org6ea8b7d)
-    35. [6.48](#org5e53f8e)
+1.  [About](#org27913d6)
+    1.  [Why Does This Exist?](#orgbf46bfc)
+2.  [Parallelism](#orgd3d922e)
+3.  [Content Hash Based Builds](#org9f2f3b2)
+4.  [Terminology](#orgdc28572)
+5.  [Projects](#org649a08e)
+    1.  [Ocaml](#org9ddb4e5)
+    2.  [Third-party](#org7b99520)
+6.  [Dependencies](#orgea56d00)
+7.  [Configuration](#org14e2b4d)
+    1.  [Configuration Variables For Projects](#org3344c3b)
+    2.  [Configuration Variables For Tests](#orgbd24be8)
+    3.  [Outputting the configuration](#orga0701b9)
+8.  [Produced Files](#orga0feeea)
+9.  [Documentation](#org93fbfd2)
+10. [Selectors](#orge3be3e9)
+11. [Changelog](#org3afc856)
+    1.  [1.0.0](#org6090dfe)
+    2.  [1.1.0](#org1087efa)
+    3.  [2.0.0](#org4837bf9)
+    4.  [3.0.0](#org0858846)
+    5.  [3.0.1](#org461e1c4)
+    6.  [3.0.2](#org04e5f50)
+    7.  [3.0.3](#org18e020c)
+    8.  [3.0.4](#org19297ee)
+    9.  [3.1.0](#orga6961dd)
+    10. [3.1.1](#orge057ee2)
+    11. [4.11](#orgd5fcaf6)
+    12. [4.12](#orge696a30)
+    13. [4.13](#org194edfe)
+    14. [5.14](#org0b4c453)
+    15. [5.15](#orgeb4f836)
+    16. [5.16](#org9529da1)
+    17. [5.17 (bug release)](#orge4b8857)
+    18. [5.18](#orgd63f583)
+    19. [5.19](#orgebed321)
+    20. [6.20](#org48d6fed)
+    21. [5.21](#orgf5dbe2e)
+    22. [5.22](#org2a4829e)
+    23. [5.23](#org14012c0)
+    24. [5.24](#org39f4d76)
+    25. [5.25](#org4ac743f)
+    26. [5.26 (bug release)](#org9cc944f)
+    27. [5.27](#orgfb401f1)
+    28. [5.28](#org5e50c40)
+    29. [5.29](#orgd927605)
+    30. [6.43](#org2b1d22b)
+    31. [6.44](#org783e4cb)
+    32. [6.45](#org653a674)
+    33. [6.46](#orgc471fc3)
+    34. [6.47](#orgb913730)
+    35. [6.48](#org7368de7)
+    36. [6.49](#org54789c2)
 
 
-<a id="org705abbd"></a>
+<a id="org27913d6"></a>
 
 # About
 
@@ -123,7 +124,7 @@
 `build/test-release`.
 
 
-<a id="org1e147e4"></a>
+<a id="orgbf46bfc"></a>
 
 ## Why Does This Exist?
 
@@ -136,7 +137,7 @@
 meant to be non-intrusive.
 
 
-<a id="org95ace50"></a>
+<a id="orgd3d922e"></a>
 
 # Parallelism
 
@@ -156,7 +157,7 @@
 error.
 
 
-<a id="orga0bd8ef"></a>
+<a id="org9f2f3b2"></a>
 
 # Content Hash Based Builds
 
@@ -176,7 +177,7 @@
 that causes a rebuild.
 
 
-<a id="org14f2d42"></a>
+<a id="orgdc28572"></a>
 
 # Terminology
 
@@ -188,7 +189,7 @@
     `library`, this applies only to ocaml projects..
 
 
-<a id="orge184885"></a>
+<a id="org649a08e"></a>
 
 # Projects
 
@@ -199,7 +200,7 @@
 and `clean`.
 
 
-<a id="org599210b"></a>
+<a id="org9ddb4e5"></a>
 
 ## Ocaml
 
@@ -250,7 +251,7 @@
 </table>
 
 
-<a id="org5025403"></a>
+<a id="org7b99520"></a>
 
 ## Third-party
 
@@ -258,7 +259,7 @@
 to have one which corresponds to the pds interface.
 
 
-<a id="orgba472af"></a>
+<a id="orgea56d00"></a>
 
 # Dependencies
 
@@ -269,7 +270,7 @@
 as a dependency in the dependents `Makefile`.
 
 
-<a id="orgfbc6674"></a>
+<a id="org14e2b4d"></a>
 
 # Configuration
 
@@ -323,7 +324,7 @@
     install = false
 
 
-<a id="orge2f869d"></a>
+<a id="org3344c3b"></a>
 
 ## Configuration Variables For Projects
 
@@ -473,7 +474,7 @@
 The same precedence applies to test builds.
 
 
-<a id="orgd259166"></a>
+<a id="orgbd24be8"></a>
 
 ## Configuration Variables For Tests
 
@@ -488,7 +489,7 @@
 3.  `tests.<project>.<option>`
 
 
-<a id="org382a124"></a>
+<a id="orga0701b9"></a>
 
 ## Outputting the configuration
 
@@ -545,7 +546,7 @@
 </table>
 
 
-<a id="orge707b46"></a>
+<a id="orga0feeea"></a>
 
 # Produced Files
 
@@ -558,7 +559,7 @@
     Ocamlrules.mk.in
 
 
-<a id="org9d84515"></a>
+<a id="org93fbfd2"></a>
 
 # Documentation
 
@@ -572,7 +573,7 @@
 `extra_makefile_lines`.
 
 
-<a id="orge5261a0"></a>
+<a id="orge3be3e9"></a>
 
 # Selectors
 
@@ -616,12 +617,12 @@
 checked, then the selector for a release.
 
 
-<a id="org96c1e30"></a>
+<a id="org3afc856"></a>
 
 # Changelog
 
 
-<a id="org9783e67"></a>
+<a id="org6090dfe"></a>
 
 ## 1.0.0
 
@@ -630,7 +631,7 @@
 -   Support running tests.
 
 
-<a id="orgc6cbf55"></a>
+<a id="org1087efa"></a>
 
 ## 1.1.0
 
@@ -639,7 +640,7 @@
     is useful if a tool needs to be built that will be used to compile a project.
 
 
-<a id="org73158b8"></a>
+<a id="org4837bf9"></a>
 
 ## 2.0.0
 
@@ -648,7 +649,7 @@
 -   Fill out the documentation more.
 
 
-<a id="org4f7c40c"></a>
+<a id="org0858846"></a>
 
 ## 3.0.0
 
@@ -656,7 +657,7 @@
     understands both installing and uninstalling projects.
 
 
-<a id="org67c028d"></a>
+<a id="org461e1c4"></a>
 
 ## 3.0.1
 
@@ -664,14 +665,14 @@
     after using hll to make the package.
 
 
-<a id="orgf7b00c6"></a>
+<a id="org04e5f50"></a>
 
 ## 3.0.2
 
 -   Expand the hll config to pass opam-linter.
 
 
-<a id="org54e2aa9"></a>
+<a id="org18e020c"></a>
 
 ## 3.0.3
 
@@ -679,7 +680,7 @@
 -   Specify which version of ocaml pds works with.
 
 
-<a id="orge538f05"></a>
+<a id="org19297ee"></a>
 
 ## 3.0.4
 
@@ -687,7 +688,7 @@
 -   Remove examples.
 
 
-<a id="org36f73d1"></a>
+<a id="orga6961dd"></a>
 
 ## 3.1.0
 
@@ -695,14 +696,14 @@
 -   Install `.cmi` files.
 
 
-<a id="org517f089"></a>
+<a id="orge057ee2"></a>
 
 ## 3.1.1
 
 -   Fix a bug in generating docs when the project has no dependencies.
 
 
-<a id="org5b01e9c"></a>
+<a id="orgd5fcaf6"></a>
 
 ## 4.11
 
@@ -713,21 +714,21 @@
     name of the output.
 
 
-<a id="orgb36a098"></a>
+<a id="orge696a30"></a>
 
 ## 4.12
 
 -   Include tests directories in formatted output.
 
 
-<a id="orgefc38b9"></a>
+<a id="org194edfe"></a>
 
 ## 4.13
 
 -   Make a better error message for the install key, which is required.
 
 
-<a id="orgd9d3d19"></a>
+<a id="org0b4c453"></a>
 
 ## 5.14
 
@@ -737,7 +738,7 @@
 -   Remove PACK support.  Building PACKs is no longer supported.
 
 
-<a id="org04bafd2"></a>
+<a id="orgeb4f836"></a>
 
 ## 5.15
 
@@ -745,14 +746,14 @@
 -   Add a changelog and back fill it.
 
 
-<a id="org8c5d0f6"></a>
+<a id="org9529da1"></a>
 
 ## 5.16
 
 -   Fix bug in cleanup logic which concatenated multiple deps
 
 
-<a id="org1757356"></a>
+<a id="orge4b8857"></a>
 
 ## 5.17 (bug release)
 
@@ -763,7 +764,7 @@
     first then the .cmo and .cmx are generated.
 
 
-<a id="org8ccbcfb"></a>
+<a id="orgd63f583"></a>
 
 ## 5.18
 
@@ -776,7 +777,7 @@
     it.  This solves it by serializing the build for those cases.
 
 
-<a id="org7a636a0"></a>
+<a id="orgebed321"></a>
 
 ## 5.19
 
@@ -784,14 +785,14 @@
     installs `.cmi` files for all `.ml` files.
 
 
-<a id="org252971c"></a>
+<a id="org48d6fed"></a>
 
 ## 6.20
 
 -   Removed the `-custom` option from default byte code builds.
 
 
-<a id="org51426d2"></a>
+<a id="orgf5dbe2e"></a>
 
 ## 5.21
 
@@ -799,7 +800,7 @@
     them to be overridden.
 
 
-<a id="orgdb47994"></a>
+<a id="org2a4829e"></a>
 
 ## 5.22
 
@@ -807,28 +808,28 @@
 -   Tests compile with `OCAML*_LINK_OPTS` just like regular builds.
 
 
-<a id="orge28c50d"></a>
+<a id="org14012c0"></a>
 
 ## 5.23
 
 -   Fix typo in PARALLEL
 
 
-<a id="org60c1f2c"></a>
+<a id="org39f4d76"></a>
 
 ## 5.24
 
 -   Build cmti files and install them for every library.
 
 
-<a id="orgeb1d233"></a>
+<a id="org4ac743f"></a>
 
 ## 5.25
 
 -   Support adding linkopts to the META file.
 
 
-<a id="org1d9ff84"></a>
+<a id="org9cc944f"></a>
 
 ## 5.26 (bug release)
 
@@ -836,14 +837,14 @@
 -   Add some tests.
 
 
-<a id="org8a0c77a"></a>
+<a id="orgfb401f1"></a>
 
 ## 5.27
 
 -   Fix bug in section name for the selector.
 
 
-<a id="orgc9518f4"></a>
+<a id="org5e50c40"></a>
 
 ## 5.28
 
@@ -851,7 +852,7 @@
     were not properly looked up.
 
 
-<a id="org01640d7"></a>
+<a id="orgd927605"></a>
 
 ## 5.29
 
@@ -860,44 +861,51 @@
     worked).
 
 
-<a id="orgc817f12"></a>
+<a id="org2b1d22b"></a>
 
 ## 6.43
 
 -   Move to content based hashing.
 
 
-<a id="orgcd3d50a"></a>
+<a id="org783e4cb"></a>
 
 ## 6.44
 
 -   Fix install directive
 
 
-<a id="org8731d09"></a>
+<a id="org653a674"></a>
 
 ## 6.45
 
 -   Better sqlite error reporting
 
 
-<a id="orge7cf45c"></a>
+<a id="orgc471fc3"></a>
 
 ## 6.46
 
 -   strftime usage that works on Alpine
 
 
-<a id="org6ea8b7d"></a>
+<a id="orgb913730"></a>
 
 ## 6.47
 
 -   Another instance of strftime
 
 
-<a id="org5e53f8e"></a>
+<a id="org7368de7"></a>
 
 ## 6.48
 
 -   Fix bugs in testing outputs.
 
+
+<a id="org54789c2"></a>
+
+## 6.49
+
+-   Improve performance 10x by batching database operations.
+
diff --git a/README.org b/README.org
--- a/README.org
+++ b/README.org
@@ -430,3 +430,5 @@
 - Another instance of strftime
 ** 6.48
 - Fix bugs in testing outputs.
+** 6.49
+- Improve performance 10x by batching database operations.
diff --git a/src/pds/pds.ml b/src/pds/pds.ml
--- a/src/pds/pds.ml
+++ b/src/pds/pds.ml
@@ -38,24 +38,32 @@
     @@ Sqlite3.exec
          db
          "create table if not exists last_run (path text primary key, hash text, modified_at text)";
-    List.iter
-      (fun (path, hash) ->
-        let hash =
-          Digest.to_hex @@ Digest.string ((Digest.to_hex @@ Digest.file path) ^ ":" ^ hash)
-        in
-        let modified_at = (Unix.stat path).Unix.st_mtime in
-        let stmt =
-          Sqlite3.prepare
-            db
-            "insert or replace into last_run (path, hash, modified_at) values (?, ?, \
-             strftime('%Y-%m-%dT%H:%M:%S', ?, 'unixepoch')) "
-        in
-        assert_rc @@ Sqlite3.bind_text stmt 1 path;
-        assert_rc @@ Sqlite3.bind_text stmt 2 hash;
-        assert_rc @@ Sqlite3.bind_double stmt 3 modified_at;
-        assert_rc @@ Sqlite3.step stmt;
+    let path_hashes_modified =
+      (List.map (fun (path, hash) ->
+           let hash =
+             Digest.to_hex @@ Digest.string ((Digest.to_hex @@ Digest.file path) ^ ":" ^ hash)
+           in
+           let modified_at = (Unix.stat path).Unix.st_mtime in
+           (path, hash, modified_at)))
+        path_hashes
+    in
+    let query =
+      "insert or replace into last_run (path, hash, modified_at) values "
+      ^ String.concat
+          ", "
+          (CCList.replicate
+             (List.length path_hashes_modified)
+             "(?, ?, strftime('%Y-%m-%dT%H:%M:%S', ?, 'unixepoch'))")
+    in
+    let stmt = Sqlite3.prepare db query in
+    List.iteri
+      (fun idx (path, hash, modified_at) ->
+        assert_rc @@ Sqlite3.bind_text stmt ((idx * 3) + 1) path;
+        assert_rc @@ Sqlite3.bind_text stmt ((idx * 3) + 2) hash;
+        assert_rc @@ Sqlite3.bind_double stmt ((idx * 3) + 3) modified_at;
         ())
-      path_hashes;
+      path_hashes_modified;
+    assert_rc @@ Sqlite3.step stmt;
     let stmt =
       Sqlite3.prepare
         db