mdk
/
formations
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

python perfs: Travail sur les TP.

This commit is contained in:
Julien Palard 2023-09-26 08:59:08 +02:00
parent 188e4bd870
commit e69a8aaf8f
Signed by: mdk
GPG Key ID: 0EFC1AC1006886F8
9 changed files with 399 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
python-perfs/README.md
View File

@ -2,6 +2,7 @@
Les slides : https://mdk.fr/python-perfs
## Description
Destinée aux développeurs Python aguerris, cette formation approche
@ -13,7 +14,7 @@ les différents moyens d’améliorer les performances d’un programme
- Savoir mesurer les performances d’un programme et identifier les goulots d’étranglement.
- Prendre conscience des impacts des différentes structures de données, de leur complexité algorithmique.
- Découvrir les différents "JIT" (*Just-In-Time* compilation) de l’écosystème Python.
- Découvrir les différents compilateurs (*Just-In-Time* et *Ahead-Of-Time*) de l’écosystème Python.
- Découvrir la variété des interpréteurs Python et leurs caractéristiques.
- Entrelacer du code natif et du Python.
@ -35,7 +36,7 @@ Les outils de mesure :
- Les outils extérieurs à Python (`time`, `hyperfine`, …).
- Configurer sa machine pour avoir des mesures reproductibles.
- Les outils de la bibliothèque standard (`cProfile`, `pstats`, `timeit`).
- Les outils tiers (`pyperf`, `snakeviz`, `Scalene`, `vprof`, …).
- Les outils tiers (`pyperf`, `snakeviz`, `Scalene`, …).
Les JIT, compilateurs, et interpréteurs tiers :
@ -45,3 +46,12 @@ Utiliser du code natif pour optimiser ponctuellement :
- Interfacer du C ou du C++ avec Python en utilisant cython.
- Rédiger un module Python en C.
## Mise en pratique
- Rédaction naïve d’un algorithme (modèle du tas de sable abélien).
- Étude de la complexité, du temps d’exécution, et des goûlots d’étranglements des implémentations.
- Implémenter des optimisations ciblées par les mesures précédentes.
- Tentative d’utiliser des structures de données spécialisées.
- Tentatives d’utiliser différents compilateurs (JIT et Ahead of Time) et interpréteurs.

42
python-perfs/examples/sandpile-array.py Normal file
View File

@ -0,0 +1,42 @@
from array import array
import sys
def show_terrain(terrain, width):
for x in range(width):
for y in range(width):
print(" ·●⬤"[terrain[x * width + y]], end="")
print()
def apply_gravity(terrain, width):
"""
$ python -m pyperf timeit --fast -s 'from examples.sandpile_array import main' 'main(10_000, False)'
...........
Mean +- std dev: 3.05 sec +- 0.12 sec
"""
while True:
did_someting = False
for x in range(width ** 2):
if terrain[x] >= 4:
div, terrain[x] = divmod(terrain[x], 4)
terrain[x - 1] += div
terrain[x + 1] += div
terrain[x - width] += div
terrain[x + width] += div
did_someting = True
if not did_someting:
return
def main(height, show=True):
width = int(height ** .5) + 1
terrain = array("Q", [0] * width ** 2)
terrain[width // 2 * width + width // 2] = height
apply_gravity(terrain, width)
if show:
show_terrain(terrain, width)
if __name__ == "__main__":
main(int(sys.argv[1]))

25
python-perfs/examples/sandpile.py
View File

@ -14,23 +14,28 @@ def show_terrain(terrain):
def apply_gravity(terrain):
"""
$ python -m pyperf timeit --fast -s 'from examples.sandpile import main' 'main(10_000, False)'
...........
Mean +- std dev: 15.4 sec +- 0.4 sec
"""
width = len(terrain)
for x in range(width):
for y in range(width):
if terrain[x][y] >= 4:
terrain[x][y] -= 4
terrain[x - 1][y] += 1
terrain[x + 1][y] += 1
terrain[x][y + 1] += 1
terrain[x][y - 1] += 1
while should_apply_gravity(terrain):
for x in range(width):
for y in range(width):
if terrain[x][y] >= 4:
terrain[x][y] -= 4
terrain[x - 1][y] += 1
terrain[x + 1][y] += 1
terrain[x][y + 1] += 1
terrain[x][y - 1] += 1
def main(height, show=True):
width = int(height ** .5) + 1
terrain = [[0] * width for _ in range(width)]
terrain[width // 2][width // 2] = height
while should_apply_gravity(terrain):
apply_gravity(terrain)
apply_gravity(terrain)
if show:
show_terrain(terrain)

44
python-perfs/examples/sandpile1.py Normal file
View File

@ -0,0 +1,44 @@
import sys
# Can handle 10k sand grains in 2s.
def show_terrain(terrain):
width = len(terrain)
for x in range(width):
for y in range(width):
print(" ·●⬤#"[min(4, terrain[x][y])], end="")
print()
def apply_gravity(terrain):
"""
$ python -m pyperf timeit --fast -s 'from examples.sandpile1 import main' 'main(10_000, False)'
...........
Mean +- std dev: 11.1 sec +- 0.2 sec
"""
width = len(terrain)
did_someting = False
for x in range(width):
for y in range(width):
if terrain[x][y] >= 4:
terrain[x][y] -= 4
terrain[x - 1][y] += 1
terrain[x + 1][y] += 1
terrain[x][y + 1] += 1
terrain[x][y - 1] += 1
did_someting = True
return did_someting
def main(height, show=True):
width = int(height ** .5) + 1
terrain = [[0] * width for _ in range(width)]
terrain[width // 2][width // 2] = height
while apply_gravity(terrain):
pass
if show:
show_terrain(terrain)
if __name__ == "__main__":
main(int(sys.argv[1]))

45
python-perfs/examples/sandpile2.py Normal file
View File

@ -0,0 +1,45 @@
import sys
# Can handle 10k sand grains in 0.4s.
def show_terrain(terrain):
width = len(terrain)
for x in range(width):
for y in range(width):
print(" ·●⬤#"[min(4, terrain[x][y])], end="")
print()
def apply_gravity(terrain):
"""
$ python -m pyperf timeit --fast -s 'from examples.sandpile2 import main' 'main(10_000, False)'
...........
Mean +- std dev: 2.42 sec +- 0.04 sec
"""
width = len(terrain)
did_someting = False
for x in range(width):
for y in range(width):
if terrain[x][y] >= 4:
div, terrain[x][y] = divmod(terrain[x][y], 4)
terrain[x - 1][y] += div
terrain[x + 1][y] += div
terrain[x][y + 1] += div
terrain[x][y - 1] += div
did_someting = True
return did_someting
def main(height, show=True):
width = int(height ** .5) + 1
terrain = [[0] * width for _ in range(width)]
terrain[width // 2][width // 2] = height
while apply_gravity(terrain):
pass
if show:
show_terrain(terrain)
if __name__ == "__main__":
main(int(sys.argv[1]))

57
python-perfs/examples/sandpile3.py Normal file
View File

@ -0,0 +1,57 @@
import sys
# Can handle 10k sand grains in 0.4s.
def show_terrain(terrain):
width = len(terrain)
for x in range(width):
for y in range(width):
print(" ·●⬤#"[min(4, terrain[x][y])], end="")
print()
def topple(terrain, x, y):
div, terrain[x][y] = divmod(terrain[x][y], 4)
terrain[x - 1][y] += div
if terrain[x-1][y] >= 4:
topple(terrain, x - 1, y)
terrain[x + 1][y] += div
terrain[x][y - 1] += div
terrain[x][y + 1] += div
def apply_gravity(terrain):
"""
$ python -m pyperf timeit --fast -s 'from examples.sandpile3 import main' 'main(10_000, False)'
...........
Mean +- std dev: 2.02 sec +- 0.04 sec
"""
width = len(terrain)
while True:
did_someting = False
for x in range(width):
for y in range(width):
if terrain[x][y] >= 4:
topple(terrain, x, y)
did_someting = True
if not did_someting:
return
def main(height, show=True):
width = int(height ** .5) + 1
terrain = [[0] * width for _ in range(width)]
terrain[width // 2][width // 2] = height
apply_gravity(terrain)
if show:
show_terrain(terrain)
if __name__ == "__main__":
main(int(sys.argv[1]))

39
python-perfs/examples/sandpile_array.py Normal file
View File

@ -0,0 +1,39 @@
from array import array
import sys
# Can handle 10k sand grains in 2s.
def show_terrain(terrain, width):
for x in range(width):
for y in range(width):
print(" ·●⬤"[terrain[x * width + y]], end="")
print()
def apply_gravity(terrain, width):
while True:
did_someting = False
for x in range(width ** 2):
if terrain[x] >= 4:
div, terrain[x] = divmod(terrain[x], 4)
terrain[x - 1] += div
terrain[x + 1] += div
terrain[x - width] += div
terrain[x + width] += div
did_someting = True
if not did_someting:
return
def main(height, show=True):
width = int(height ** .5) + 1
terrain = array("Q", [0] * width ** 2)
terrain[width // 2 * width + width // 2] = height
apply_gravity(terrain, width)
if show:
show_terrain(terrain, width)
if __name__ == "__main__":
main(int(sys.argv[1]))

129
python-perfs/examples/sandpile_numpy.py Normal file
View File

@ -0,0 +1,129 @@
import numba
import numpy as np
from time import perf_counter
import sys
def show_terrain(terrain, width):
for x in range(width):
for y in range(width):
print(" ·●⬤"[int(terrain[x, y])], end="")
print()
def apply_gravity(terrain):
"""
$ python -m pyperf timeit --fast -s 'from examples.sandpile_numpy import main' 'main(10_000, False)'
...........
Mean +- std dev: 799 ms +- 42 ms
"""
while True:
tops = terrain > 3
if not np.any(tops):
return
terrain[tops] -= 4
terrain[1:, :][tops[:-1, :]] += 1
terrain[:-1, :][tops[1:, :]] += 1
terrain[:, 1:][tops[:, :-1]] += 1
terrain[:, :-1][tops[:, 1:]] += 1
def apply_gravity(terrain):
"""
$ python -m pyperf timeit --fast -s 'from examples.sandpile_numpy import main' 'main(10_000, False)'
...........
Mean +- std dev: 356 ms +- 35 ms
"""
while True:
tumbled, terrain = np.divmod(terrain, 4)
if not np.any(tumbled):
return terrain
terrain[1:, :] += tumbled[:-1, :]
terrain[:-1, :] += tumbled[1:, :]
terrain[:, 1:] += tumbled[:, :-1]
terrain[:, :-1] += tumbled[:, 1:]
@numba.njit(numba.void(numba.int64[:,:]))
def apply_gravity(terrain):
"""
$ python -m pyperf timeit --fast -s 'from examples.sandpile_numpy import main' 'main(10_000, False)'
...........
Mean +- std dev: 19.2 ms +- 0.3 ms
"""
shape = np.shape(terrain)
while True:
did_someting = False
for x, y in np.ndindex(shape):
if terrain[x, y] >= 4:
div, terrain[x, y] = divmod(terrain[x][y], 4)
terrain[x - 1][y] += div
terrain[x + 1][y] += div
terrain[x][y + 1] += div
terrain[x][y - 1] += div
did_someting = True
if not did_someting:
return
@numba.njit(numba.void(numba.int64[:,:]))
def apply_gravity(terrain):
"""
$ python -m pyperf timeit --fast -s 'from examples.sandpile_numpy import main' 'main(10_000, False)'
...........
Mean +- std dev: 100 ms +- 5 ms
"""
shape = np.shape(terrain)
while True:
did_someting = False
for x, y in np.ndindex(shape):
if terrain[x, y] >= 4:
terrain[x,y] -= 4
terrain[x - 1][y] += 1
terrain[x + 1][y] += 1
terrain[x][y + 1] += 1
terrain[x][y - 1] += 1
did_someting = True
if not did_someting:
return
@numba.njit(numba.void(numba.int64[:,:]))
def apply_gravity(terrain):
"""Can handle 10k sand grains in 1.5s."""
shape = np.shape(terrain)
while True:
did_someting = False
for x, y in np.ndindex(shape):
if terrain[x, y] >= 4000:
div, terrain[x, y] = divmod(terrain[x][y], 4)
terrain[x - 1][y] += div
terrain[x + 1][y] += div
terrain[x][y + 1] += div
terrain[x][y - 1] += div
did_someting = True
elif terrain[x, y] >= 4:
terrain[x,y] -= 4
terrain[x - 1][y] += 1
terrain[x + 1][y] += 1
terrain[x][y + 1] += 1
terrain[x][y - 1] += 1
did_someting = True
if not did_someting:
return
def main(height, show=True):
width = int(height**0.5) + 1
terrain = np.zeros((width, width), dtype=np.int64)
terrain[width // 2, width // 2] = height
begin = perf_counter()
apply_gravity(terrain)
if show:
show_terrain(terrain, width)
if __name__ == "__main__":
main(int(sys.argv[1]))

16
python-perfs/examples/sandpilelib.py Normal file
View File

@ -0,0 +1,16 @@
import numpy as np
#pythran export apply_gravity(int[][])
def apply_gravity(terrain):
"""Can handle 10k sand grains in 800ms."""
while True:
tops = terrain > 3
if not np.any(tops):
return
terrain[tops] -= 4
terrain[1:, :][tops[:-1, :]] += 1
terrain[:-1, :][tops[1:, :]] += 1
terrain[:, 1:][tops[:, :-1]] += 1
terrain[:, :-1][tops[:, 1:]] += 1