From e69a8aaf8fcbefc57c1637ef3e6b1731eb8fa22e Mon Sep 17 00:00:00 2001
From: Julien Palard <julien@palard.fr>
Date: Tue, 26 Sep 2023 08:59:08 +0200
Subject: [PATCH] python perfs: Travail sur les TP.

---
 python-perfs/README.md                  |  14 ++-
 python-perfs/examples/sandpile-array.py |  42 ++++++++
 python-perfs/examples/sandpile.py       |  25 +++--
 python-perfs/examples/sandpile1.py      |  44 ++++++++
 python-perfs/examples/sandpile2.py      |  45 +++++++++
 python-perfs/examples/sandpile3.py      |  57 +++++++++++
 python-perfs/examples/sandpile_array.py |  39 +++++++
 python-perfs/examples/sandpile_numpy.py | 129 ++++++++++++++++++++++++
 python-perfs/examples/sandpilelib.py    |  16 +++
 9 files changed, 399 insertions(+), 12 deletions(-)
 create mode 100644 python-perfs/examples/sandpile-array.py
 create mode 100644 python-perfs/examples/sandpile1.py
 create mode 100644 python-perfs/examples/sandpile2.py
 create mode 100644 python-perfs/examples/sandpile3.py
 create mode 100644 python-perfs/examples/sandpile_array.py
 create mode 100644 python-perfs/examples/sandpile_numpy.py
 create mode 100644 python-perfs/examples/sandpilelib.py

diff --git a/python-perfs/README.md b/python-perfs/README.md
index 6102715..1c6ea94 100644
--- a/python-perfs/README.md
+++ b/python-perfs/README.md
@@ -2,6 +2,7 @@
 
 Les slides : https://mdk.fr/python-perfs
 
+
 ## Description
 
 Destinée aux développeurs Python aguerris, cette formation approche
@@ -13,7 +14,7 @@ les différents moyens d’améliorer les performances d’un programme
 
 - Savoir mesurer les performances d’un programme et identifier les goulots d’étranglement.
 - Prendre conscience des impacts des différentes structures de données, de leur complexité algorithmique.
-- Découvrir les différents "JIT" (*Just-In-Time* compilation) de l’écosystème Python.
+- Découvrir les différents compilateurs (*Just-In-Time* et *Ahead-Of-Time*) de l’écosystème Python.
 - Découvrir la variété des interpréteurs Python et leurs caractéristiques.
 - Entrelacer du code natif et du Python.
 
@@ -35,7 +36,7 @@ Les outils de mesure :
 - Les outils extérieurs à Python (`time`, `hyperfine`, …).
 - Configurer sa machine pour avoir des mesures reproductibles.
 - Les outils de la bibliothèque standard (`cProfile`, `pstats`, `timeit`).
-- Les outils tiers (`pyperf`, `snakeviz`, `Scalene`, `vprof`, …).
+- Les outils tiers (`pyperf`, `snakeviz`, `Scalene`, …).
 
 Les JIT, compilateurs, et interpréteurs tiers :
 
@@ -45,3 +46,12 @@ Utiliser du code natif pour optimiser ponctuellement :
 
 - Interfacer du C ou du C++ avec Python en utilisant cython.
 - Rédiger un module Python en C.
+
+
+## Mise en pratique
+
+- Rédaction naïve d’un algorithme (modèle du tas de sable abélien).
+- Étude de la complexité, du temps d’exécution, et des goûlots d’étranglements des implémentations.
+- Implémenter des optimisations ciblées par les mesures précédentes.
+- Tentative d’utiliser des structures de données spécialisées.
+- Tentatives d’utiliser différents compilateurs (JIT et Ahead of Time) et interpréteurs.
diff --git a/python-perfs/examples/sandpile-array.py b/python-perfs/examples/sandpile-array.py
new file mode 100644
index 0000000..1ccd2f1
--- /dev/null
+++ b/python-perfs/examples/sandpile-array.py
@@ -0,0 +1,42 @@
+from array import array
+import sys
+
+
+def show_terrain(terrain, width):
+    for x in range(width):
+        for y in range(width):
+            print(" ·●⬤"[terrain[x * width + y]], end="")
+        print()
+
+
+def apply_gravity(terrain, width):
+    """
+    $ python -m pyperf timeit --fast -s 'from examples.sandpile_array import main' 'main(10_000, False)'
+    ...........
+    Mean +- std dev: 3.05 sec +- 0.12 sec
+    """
+    while True:
+        did_someting = False
+        for x in range(width ** 2):
+            if terrain[x] >= 4:
+                div, terrain[x] = divmod(terrain[x], 4)
+                terrain[x - 1] += div
+                terrain[x + 1] += div
+                terrain[x - width] += div
+                terrain[x + width] += div
+                did_someting = True
+        if not did_someting:
+            return
+
+
+def main(height, show=True):
+    width = int(height ** .5) + 1
+    terrain = array("Q", [0] * width ** 2)
+    terrain[width // 2 * width + width // 2] = height
+    apply_gravity(terrain, width)
+    if show:
+        show_terrain(terrain, width)
+
+
+if __name__ == "__main__":
+    main(int(sys.argv[1]))
diff --git a/python-perfs/examples/sandpile.py b/python-perfs/examples/sandpile.py
index 27ecef7..97d8e2a 100644
--- a/python-perfs/examples/sandpile.py
+++ b/python-perfs/examples/sandpile.py
@@ -14,23 +14,28 @@ def show_terrain(terrain):
 
 
 def apply_gravity(terrain):
+    """
+    $ python -m pyperf timeit --fast -s 'from examples.sandpile import main' 'main(10_000, False)'
+    ...........
+    Mean +- std dev: 15.4 sec +- 0.4 sec
+    """
     width = len(terrain)
-    for x in range(width):
-        for y in range(width):
-            if terrain[x][y] >= 4:
-                terrain[x][y] -= 4
-                terrain[x - 1][y] += 1
-                terrain[x + 1][y] += 1
-                terrain[x][y + 1] += 1
-                terrain[x][y - 1] += 1
+    while should_apply_gravity(terrain):
+        for x in range(width):
+            for y in range(width):
+                if terrain[x][y] >= 4:
+                    terrain[x][y] -= 4
+                    terrain[x - 1][y] += 1
+                    terrain[x + 1][y] += 1
+                    terrain[x][y + 1] += 1
+                    terrain[x][y - 1] += 1
 
 
 def main(height, show=True):
     width = int(height ** .5) + 1
     terrain = [[0] * width for _ in range(width)]
     terrain[width // 2][width // 2] = height
-    while should_apply_gravity(terrain):
-        apply_gravity(terrain)
+    apply_gravity(terrain)
     if show:
         show_terrain(terrain)
 
diff --git a/python-perfs/examples/sandpile1.py b/python-perfs/examples/sandpile1.py
new file mode 100644
index 0000000..e5ddb46
--- /dev/null
+++ b/python-perfs/examples/sandpile1.py
@@ -0,0 +1,44 @@
+import sys
+
+# Can handle 10k sand grains in 2s.
+
+def show_terrain(terrain):
+    width = len(terrain)
+    for x in range(width):
+        for y in range(width):
+            print(" ·●⬤#"[min(4, terrain[x][y])], end="")
+        print()
+
+
+def apply_gravity(terrain):
+    """
+    $ python -m pyperf timeit --fast -s 'from examples.sandpile1 import main' 'main(10_000, False)'
+    ...........
+    Mean +- std dev: 11.1 sec +- 0.2 sec
+    """
+    width = len(terrain)
+    did_someting = False
+    for x in range(width):
+        for y in range(width):
+            if terrain[x][y] >= 4:
+                terrain[x][y] -= 4
+                terrain[x - 1][y] += 1
+                terrain[x + 1][y] += 1
+                terrain[x][y + 1] += 1
+                terrain[x][y - 1] += 1
+                did_someting = True
+    return did_someting
+
+
+def main(height, show=True):
+    width = int(height ** .5) + 1
+    terrain = [[0] * width for _ in range(width)]
+    terrain[width // 2][width // 2] = height
+    while apply_gravity(terrain):
+        pass
+    if show:
+        show_terrain(terrain)
+
+
+if __name__ == "__main__":
+    main(int(sys.argv[1]))
diff --git a/python-perfs/examples/sandpile2.py b/python-perfs/examples/sandpile2.py
new file mode 100644
index 0000000..cfd5ab2
--- /dev/null
+++ b/python-perfs/examples/sandpile2.py
@@ -0,0 +1,45 @@
+import sys
+
+# Can handle 10k sand grains in 0.4s.
+
+
+def show_terrain(terrain):
+    width = len(terrain)
+    for x in range(width):
+        for y in range(width):
+            print(" ·●⬤#"[min(4, terrain[x][y])], end="")
+        print()
+
+
+def apply_gravity(terrain):
+    """
+    $ python -m pyperf timeit --fast -s 'from examples.sandpile2 import main' 'main(10_000, False)'
+    ...........
+    Mean +- std dev: 2.42 sec +- 0.04 sec
+    """
+    width = len(terrain)
+    did_someting = False
+    for x in range(width):
+        for y in range(width):
+            if terrain[x][y] >= 4:
+                div, terrain[x][y] = divmod(terrain[x][y], 4)
+                terrain[x - 1][y] += div
+                terrain[x + 1][y] += div
+                terrain[x][y + 1] += div
+                terrain[x][y - 1] += div
+                did_someting = True
+    return did_someting
+
+
+def main(height, show=True):
+    width = int(height ** .5) + 1
+    terrain = [[0] * width for _ in range(width)]
+    terrain[width // 2][width // 2] = height
+    while apply_gravity(terrain):
+        pass
+    if show:
+        show_terrain(terrain)
+
+
+if __name__ == "__main__":
+    main(int(sys.argv[1]))
diff --git a/python-perfs/examples/sandpile3.py b/python-perfs/examples/sandpile3.py
new file mode 100644
index 0000000..4f1b74e
--- /dev/null
+++ b/python-perfs/examples/sandpile3.py
@@ -0,0 +1,57 @@
+import sys
+
+# Can handle 10k sand grains in 0.4s.
+
+
+def show_terrain(terrain):
+    width = len(terrain)
+    for x in range(width):
+        for y in range(width):
+            print(" ·●⬤#"[min(4, terrain[x][y])], end="")
+        print()
+
+
+def topple(terrain, x, y):
+    div, terrain[x][y] = divmod(terrain[x][y], 4)
+
+    terrain[x - 1][y] += div
+
+    if terrain[x-1][y] >= 4:
+        topple(terrain, x - 1, y)
+
+    terrain[x + 1][y] += div
+    terrain[x][y - 1] += div
+    terrain[x][y + 1] += div
+
+
+
+def apply_gravity(terrain):
+    """
+    $ python -m pyperf timeit --fast -s 'from examples.sandpile3 import main' 'main(10_000, False)'
+    ...........
+    Mean +- std dev: 2.02 sec +- 0.04 sec
+
+    """
+    width = len(terrain)
+    while True:
+        did_someting = False
+        for x in range(width):
+            for y in range(width):
+                if terrain[x][y] >= 4:
+                    topple(terrain, x, y)
+                    did_someting = True
+        if not did_someting:
+            return
+
+
+def main(height, show=True):
+    width = int(height ** .5) + 1
+    terrain = [[0] * width for _ in range(width)]
+    terrain[width // 2][width // 2] = height
+    apply_gravity(terrain)
+    if show:
+        show_terrain(terrain)
+
+
+if __name__ == "__main__":
+    main(int(sys.argv[1]))
diff --git a/python-perfs/examples/sandpile_array.py b/python-perfs/examples/sandpile_array.py
new file mode 100644
index 0000000..78afb51
--- /dev/null
+++ b/python-perfs/examples/sandpile_array.py
@@ -0,0 +1,39 @@
+from array import array
+import sys
+
+# Can handle 10k sand grains in 2s.
+
+
+def show_terrain(terrain, width):
+    for x in range(width):
+        for y in range(width):
+            print(" ·●⬤"[terrain[x * width + y]], end="")
+        print()
+
+
+def apply_gravity(terrain, width):
+    while True:
+        did_someting = False
+        for x in range(width ** 2):
+            if terrain[x] >= 4:
+                div, terrain[x] = divmod(terrain[x], 4)
+                terrain[x - 1] += div
+                terrain[x + 1] += div
+                terrain[x - width] += div
+                terrain[x + width] += div
+                did_someting = True
+        if not did_someting:
+            return
+
+
+def main(height, show=True):
+    width = int(height ** .5) + 1
+    terrain = array("Q", [0] * width ** 2)
+    terrain[width // 2 * width + width // 2] = height
+    apply_gravity(terrain, width)
+    if show:
+        show_terrain(terrain, width)
+
+
+if __name__ == "__main__":
+    main(int(sys.argv[1]))
diff --git a/python-perfs/examples/sandpile_numpy.py b/python-perfs/examples/sandpile_numpy.py
new file mode 100644
index 0000000..d1eb824
--- /dev/null
+++ b/python-perfs/examples/sandpile_numpy.py
@@ -0,0 +1,129 @@
+import numba
+import numpy as np
+from time import perf_counter
+import sys
+
+
+def show_terrain(terrain, width):
+    for x in range(width):
+        for y in range(width):
+            print(" ·●⬤"[int(terrain[x, y])], end="")
+        print()
+
+
+def apply_gravity(terrain):
+    """
+    $ python -m pyperf timeit --fast -s 'from examples.sandpile_numpy import main' 'main(10_000, False)'
+    ...........
+    Mean +- std dev: 799 ms +- 42 ms
+    """
+    while True:
+        tops = terrain > 3
+        if not np.any(tops):
+            return
+        terrain[tops] -= 4
+        terrain[1:, :][tops[:-1, :]] += 1
+        terrain[:-1, :][tops[1:, :]] += 1
+        terrain[:, 1:][tops[:, :-1]] += 1
+        terrain[:, :-1][tops[:, 1:]] += 1
+
+
+def apply_gravity(terrain):
+    """
+    $ python -m pyperf timeit --fast -s 'from examples.sandpile_numpy import main' 'main(10_000, False)'
+    ...........
+    Mean +- std dev: 356 ms +- 35 ms
+    """
+    while True:
+        tumbled, terrain = np.divmod(terrain, 4)
+        if not np.any(tumbled):
+            return terrain
+        terrain[1:, :] += tumbled[:-1, :]
+        terrain[:-1, :] += tumbled[1:, :]
+        terrain[:, 1:] += tumbled[:, :-1]
+        terrain[:, :-1] += tumbled[:, 1:]
+
+
+@numba.njit(numba.void(numba.int64[:,:]))
+def apply_gravity(terrain):
+    """
+    $ python -m pyperf timeit --fast -s 'from examples.sandpile_numpy import main' 'main(10_000, False)'
+    ...........
+    Mean +- std dev: 19.2 ms +- 0.3 ms
+    """
+    shape = np.shape(terrain)
+    while True:
+        did_someting = False
+        for x, y in np.ndindex(shape):
+            if terrain[x, y] >= 4:
+                div, terrain[x, y] = divmod(terrain[x][y], 4)
+                terrain[x - 1][y] += div
+                terrain[x + 1][y] += div
+                terrain[x][y + 1] += div
+                terrain[x][y - 1] += div
+                did_someting = True
+
+        if not did_someting:
+            return
+
+@numba.njit(numba.void(numba.int64[:,:]))
+def apply_gravity(terrain):
+    """
+    $ python -m pyperf timeit --fast -s 'from examples.sandpile_numpy import main' 'main(10_000, False)'
+    ...........
+    Mean +- std dev: 100 ms +- 5 ms
+    """
+    shape = np.shape(terrain)
+    while True:
+        did_someting = False
+        for x, y in np.ndindex(shape):
+            if terrain[x, y] >= 4:
+                terrain[x,y] -= 4
+                terrain[x - 1][y] += 1
+                terrain[x + 1][y] += 1
+                terrain[x][y + 1] += 1
+                terrain[x][y - 1] += 1
+                did_someting = True
+
+        if not did_someting:
+            return
+
+@numba.njit(numba.void(numba.int64[:,:]))
+def apply_gravity(terrain):
+    """Can handle 10k sand grains in 1.5s."""
+    shape = np.shape(terrain)
+    while True:
+        did_someting = False
+        for x, y in np.ndindex(shape):
+            if terrain[x, y] >= 4000:
+                div, terrain[x, y] = divmod(terrain[x][y], 4)
+                terrain[x - 1][y] += div
+                terrain[x + 1][y] += div
+                terrain[x][y + 1] += div
+                terrain[x][y - 1] += div
+                did_someting = True
+            elif terrain[x, y] >= 4:
+                terrain[x,y] -= 4
+                terrain[x - 1][y] += 1
+                terrain[x + 1][y] += 1
+                terrain[x][y + 1] += 1
+                terrain[x][y - 1] += 1
+                did_someting = True
+
+        if not did_someting:
+            return
+
+
+def main(height, show=True):
+    width = int(height**0.5) + 1
+    terrain = np.zeros((width, width), dtype=np.int64)
+    terrain[width // 2, width // 2] = height
+    begin = perf_counter()
+    apply_gravity(terrain)
+
+    if show:
+        show_terrain(terrain, width)
+
+
+if __name__ == "__main__":
+    main(int(sys.argv[1]))
diff --git a/python-perfs/examples/sandpilelib.py b/python-perfs/examples/sandpilelib.py
new file mode 100644
index 0000000..c28b235
--- /dev/null
+++ b/python-perfs/examples/sandpilelib.py
@@ -0,0 +1,16 @@
+import numpy as np
+
+
+#pythran export apply_gravity(int[][])
+
+def apply_gravity(terrain):
+    """Can handle 10k sand grains in 800ms."""
+    while True:
+        tops = terrain > 3
+        if not np.any(tops):
+            return
+        terrain[tops] -= 4
+        terrain[1:, :][tops[:-1, :]] += 1
+        terrain[:-1, :][tops[1:, :]] += 1
+        terrain[:, 1:][tops[:, :-1]] += 1
+        terrain[:, :-1][tops[:, 1:]] += 1