Commit f426f0fa authored by Pierre Augier's avatar Pierre Augier
Browse files

Reexecute notebooks

parent cec3d7d4eb13
Pipeline #29854 passed with stage
in 9 minutes and 53 seconds
%% Cell type:markdown id: tags:
# Benchmark in Jupyter with the `jit` decorator
%% Cell type:code id: tags:
``` python
import numpy as np
from transonic import jit
def fxfy(ft, fn, theta):
sin_theta = np.sin(theta)
cos_theta = np.cos(theta)
fx = cos_theta * ft - sin_theta * fn
fy = sin_theta * ft + cos_theta * fn
return fx, fy
def fxfy_loops(ft, fn, theta):
n0 = theta.size
fx = np.empty_like(ft)
fy = np.empty_like(fn)
for index in range(n0):
sin_theta = np.sin(theta[index])
cos_theta = np.cos(theta[index])
fx[index] = cos_theta * ft[index] - sin_theta * fn[index]
fy[index] = sin_theta * ft[index] + cos_theta * fn[index]
return fx, fy
fxfy_pythran = jit(backend="pythran", native=True, xsimd=True)(fxfy)
fxfy_numba = jit(backend="numba")(fxfy)
fxfy_loops_pythran = jit(backend="pythran", native=True, xsimd=True)(fxfy_loops)
fxfy_loops_numba = jit(backend="numba")(fxfy_loops)
```
%% Cell type:code id: tags:
``` python
from transonic import wait_for_all_extensions
from transonic.util import print_versions, timeit_verbose
print_versions()
```
%%%% Output: stream
Transonic 0.4.11
Pythran 0.10.0
Numba 0.54.1
Cython 0.29.24
%% Cell type:code id: tags:
``` python
theta = np.linspace(0, 2 * np.pi, 10000)
ft = 2.5 * theta
fv = 1.5 * theta
loc = locals()
out = fxfy(ft, fv, theta)
out_loops = fxfy_loops(ft, fv, theta)
assert np.allclose(out, out_loops)
```
%% Cell type:code id: tags:
``` python
# warmup
fxfy_pythran(ft, fv, theta)
fxfy_loops_pythran(ft, fv, theta)
fxfy_numba(ft, fv, theta)
fxfy_loops_numba(ft, fv, theta)
wait_for_all_extensions()
```
%%%% Output: stream
Wait for all extensions
%% Cell type:code id: tags:
``` python
norm = timeit_verbose("fxfy(ft, fv, theta)", globals=loc)
for backend in ("numba", "pythran"):
timeit_verbose(f"fxfy_{backend}(ft, fv, theta)", globals=loc, norm=norm)
timeit_verbose(
f"fxfy_loops_{backend}(ft, fv, theta)", globals=loc, norm=norm
)
```
%%%% Output: stream
fxfy : 1.000 * norm
norm = 6.90e-04 s
fxfy_numba : 0.952 * norm
fxfy_loops_numba : 0.776 * norm
fxfy_pythran : 0.152 * norm
fxfy_loops_pythran : 0.784 * norm
fxfy : 1 * norm
norm = 0.000208 s
fxfy_numba : 0.927 * norm
fxfy_loops_numba : 0.808 * norm
fxfy_pythran : 0.901 * norm
fxfy_loops_pythran : 0.797 * norm
......
%% Cell type:markdown id: tags:
## Ipython and the "compile_at_import" mode
Here, we show how easy it is to work with modules using Pythran from IPython. We also show a little bit of how it works internally.
%% Cell type:code id: tags:
``` python
import time
from pathlib import Path
from transonic import set_compile_at_import
# an internal object to get more information on what is going on under the hood
from transonic.aheadoftime import modules
```
%% Cell type:markdown id: tags:
We define a small function to display the state of a Transonic object representing a module:
%% Cell type:code id: tags:
``` python
def get_ts_mod_display_state(name):
ts = modules[name]
print(
"(is_transpiled, is_compiling, is_compiled) =",
(ts.is_transpiled, ts.is_compiling, ts.is_compiled),
)
return ts
```
%% Cell type:markdown id: tags:
We first write in a small Python module which defines a function using Transonic. We are going to run this module and to modify it during the Ipython session.
%% Cell type:code id: tags:
``` python
src = """
from transonic import boost
@boost
def func(n: int):
return 2 * n
if __name__ == "__main__":
print("result func(1):", func(1))
"""
with open("tmp.py", "w") as file:
file.write(src)
```
%% Cell type:markdown id: tags:
Let's first clean all Transonic cache files...
%% Cell type:code id: tags:
``` python
paths = Path("__pythran__").glob("tmp*.*")
for path in paths:
path.unlink()
```
%% Cell type:markdown id: tags:
We first use Transonic in the default mode for which Pythran is not used at run time.
%% Cell type:code id: tags:
``` python
set_compile_at_import(False)
```
%% Cell type:code id: tags:
``` python
run tmp.py
```
%%%% Output: display_data
%%%% Output: stream
Module /home/pierre/Dev/transonic/doc/ipynb/executed/tmp.py has not been compiled for Transonic-Pythran
%%%% Output: stream
result func(1): 2
%% Cell type:code id: tags:
``` python
run tmp.py
```
%%%% Output: stream
result func(1): 2
%% Cell type:markdown id: tags:
Let's now switch on the Transonic mode with Pythran compilations at import time.
%% Cell type:code id: tags:
``` python
set_compile_at_import(True)
```
%% Cell type:markdown id: tags:
We then rerun the tmp.py module:
%% Cell type:code id: tags:
``` python
%run tmp.py
```
%%%% Output: stream
Running transonic on file /home/pierre/Dev/transonic/doc/ipynb/executed/tmp.py... WARNING 1 files created or updated needs to be pythranized
Running transonic on file /home/pierre/Dev/transonic/doc/ipynb/executed/tmp.py...
%%%% Output: stream
1 files created or updated needs to be pythranized
Schedule pythranization: tmp.py
%%%% Output: stream
Done!
Launching Pythran to compile a new extension...
result func(1): 2
%% Cell type:code id: tags:
``` python
ls --color=never __pythran__
```
%%%% Output: stream
Pythranizing /home/pierre/Dev/transonic/doc/ipynb/executed/__pythran__/tmp.py
tmp.py tmp.pythran tmp_fdadadf0c3880ac863b8957a4d5dac41.lock __pycache__/
%% Cell type:markdown id: tags:
Note that we used `ls --color=never` just because colors in the terminal are not well rendered in html produced by Jupyter nbconvert...
%% Cell type:code id: tags:
``` python
ts_mod = get_ts_mod_display_state("tmp")
```
%%%% Output: stream
(is_transpiled, is_compiling, is_compiled) = (True, True, False)
%% Cell type:markdown id: tags:
We see that Transonic created a Pythran file at import and launch the compilation. We can still rerun the module and it works as without Transonic:
%% Cell type:code id: tags:
``` python
%run tmp.py
```
%%%% Output: stream
result func(1): 2
%% Cell type:markdown id: tags:
While it's compiling, let's wait and call the function.
%% Cell type:code id: tags:
``` python
while ts_mod.is_compiling:
print(func(1), end=",")
time.sleep(1)
```
%%%% Output: stream
2,2,2,2,2,
2,2,
%%%% Output: stream
Pythran stderr:
INFO: sys file exists: /home/pierre/.pyenv/versions/3.9.7/lib/python3.9/site-packages/pythran/pythran.cfg
INFO: platform file exists: /home/pierre/.pyenv/versions/3.9.7/lib/python3.9/site-packages/pythran/pythran-linux.cfg
INFO: sys file exists: /home/pierre/.virtualenvs/env_transonic/lib/python3.8/site-packages/pythran/pythran.cfg
INFO: platform file exists: /home/pierre/.virtualenvs/env_transonic/lib/python3.8/site-packages/pythran/pythran-linux.cfg
INFO: user file exists: /home/pierre/.pythranrc
INFO: pythranrc section [pythran] is valid and options are correct
INFO: pythranrc section [typing] is valid and options are correct
INFO: pythranrc section [compiler] is valid and options are correct
INFO: Generated module: tmp_fdadadf0c3880ac863b8957a4d5dac41
INFO: Output: tmp_fdadadf0c3880ac863b8957a4d5dac41.tmp
%%%% Output: stream
File /home/pierre/Dev/transonic/doc/ipynb/executed/__pythran__/tmp_fdadadf0c3880ac863b8957a4d5dac41.cpython-39-x86_64-linux-gnu.so created by pythran
File /home/pierre/Dev/transonic/doc/ipynb/executed/__pythran__/tmp_fdadadf0c3880ac863b8957a4d5dac41.cpython-38-x86_64-linux-gnu.so created by pythran
Pythran stdout:
running build_ext
new_compiler returns <class 'distutils.unixccompiler.UnixCCompiler'>
building 'tmp_fdadadf0c3880ac863b8957a4d5dac41' extension
C compiler: clang++-6.0 -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -fPIC
C compiler: clang++-10 -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC
creating /tmp/tmpi559crhy/tmp
compile options: '-DENABLE_PYTHON_MODULE -D__PYTHRAN__=3 -DPYTHRAN_BLAS_BLAS -I/home/pierre/.pyenv/versions/3.9.7/lib/python3.9/site-packages/pythran -I/home/pierre/.pyenv/versions/3.9.7/lib/python3.9/site-packages/pythran/pythonic/patch -I/home/pierre/.pyenv/versions/3.9.7/lib/python3.9/site-packages/numpy/core/include -I/usr/local/include -I/usr/include -I/home/pierre/.pyenv/versions/3.9.7/include -I/home/pierre/.pyenv/versions/3.9.7/include/python3.9 -c'
creating /tmp/tmp4sxtyl63/tmp
compile options: '-DENABLE_PYTHON_MODULE -D__PYTHRAN__=3 -DPYTHRAN_BLAS_BLAS -I/home/pierre/.virtualenvs/env_transonic/lib/python3.8/site-packages/pythran -I/home/pierre/.virtualenvs/env_transonic/lib/python3.8/site-packages/pythran/pythonic/patch -I/home/pierre/.virtualenvs/env_transonic/lib/python3.8/site-packages/numpy/core/include -I/usr/local/include -I/usr/include -I/home/pierre/.virtualenvs/env_transonic/include -I/home/pierre/.virtualenvs/env_transonic/include -I/usr/include/python3.8 -c'
extra options: '-std=c++11 -fno-math-errno -fvisibility=hidden -fno-wrapv -Wno-unused-function -Wno-int-in-bool-context -Wno-unknown-warning-option'
clang++-6.0: /tmp/tmpft_3ekmp.cpp
clang++-6.0 -pthread -shared -L/home/pierre/.pyenv/versions/3.9.7/lib -L/home/pierre/.pyenv/versions/3.9.7/lib /tmp/tmpi559crhy/tmp/tmpft_3ekmp.o -L/usr/lib/x86_64-linux-gnu -lblas -lblas -o /tmp/tmpz_sui43s/tmp_fdadadf0c3880ac863b8957a4d5dac41.cpython-39-x86_64-linux-gnu.so -fvisibility=hidden -Wl,-strip-all
clang++-10: /tmp/tmpj8866sl1.cpp
clang++-10 -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fwrapv -O2 -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 /tmp/tmp4sxtyl63/tmp/tmpj8866sl1.o -lblas -lblas -o /tmp/tmpbucwda3c/tmp_fdadadf0c3880ac863b8957a4d5dac41.cpython-38-x86_64-linux-gnu.so -fvisibility=hidden -Wl,-strip-all
2,
%% Cell type:markdown id: tags:
Ok so here, we see that Pythran compilations are quite long! In particular compared to Numba, PyPy or Julia JIT compilations! But after these few seconds, it should very efficient...
%% Cell type:code id: tags:
``` python
ts_mod = get_ts_mod_display_state("tmp")
```
%%%% Output: stream
(is_transpiled, is_compiling, is_compiled) = (True, False, True)
%% Cell type:markdown id: tags:
Now the compilation is done. Let's check that there is a new extension file:
%% Cell type:code id: tags:
``` python
ls --color=never __pythran__
```
%%%% Output: stream
tmp.py
tmp.pythran
tmp_fdadadf0c3880ac863b8957a4d5dac41.cpython-39-x86_64-linux-gnu.so
tmp_fdadadf0c3880ac863b8957a4d5dac41.cpython-38-x86_64-linux-gnu.so
__pycache__/
%% Cell type:code id: tags:
``` python
print("use the compiled functions (now it should be faster):", func(1))
```
%%%% Output: stream
use the compiled functions (now it should be faster): 2
%% Cell type:code id: tags:
``` python
%run tmp.py
```
%%%% Output: stream
result func(1): 2
%% Cell type:code id: tags:
``` python
print("use the compiled functions again:", func(1))
```
%%%% Output: stream
use the compiled functions again: 2
%% Cell type:markdown id: tags:
Let's update the code of the module and see how Transonic is going to update the Pythran file and the extension!
%% Cell type:code id: tags:
``` python
src1 = """
from transonic import boost
@boost
def func(n: int):
return 4 * n
if __name__ == "__main__":
print("result func(1):", func(1))
"""
with open("tmp.py", "w") as file:
file.write(src1)
```
%% Cell type:code id: tags:
``` python
%run tmp.py
```
%%%% Output: stream
Running transonic on file /home/pierre/Dev/transonic/doc/ipynb/executed/tmp.py... WARNING 1 files created or updated needs to be pythranized
Running transonic on file /home/pierre/Dev/transonic/doc/ipynb/executed/tmp.py...
%%%% Output: stream
1 files created or updated needs to be pythranized
Schedule pythranization: tmp.py
%%%% Output: stream
Done!
Launching Pythran to compile a new extension...
result func(1): 4
%% Cell type:code id: tags:
``` python
%run tmp.py
```
%%%% Output: stream
result func(1): 4
%% Cell type:code id: tags:
``` python
ts_mod = get_ts_mod_display_state("tmp")
```
%%%% Output: stream
(is_transpiled, is_compiling, is_compiled) = (True, True, False)
%% Cell type:code id: tags:
``` python
while ts_mod.is_compiling:
print(func(1), end=",")
time.sleep(1)
```
%%%% Output: stream
4,Pythranizing /home/pierre/Dev/transonic/doc/ipynb/executed/__pythran__/tmp.py
4,4,4,
4,4,
%%%% Output: stream
Pythran stderr:
INFO: sys file exists: /home/pierre/.pyenv/versions/3.9.7/lib/python3.9/site-packages/pythran/pythran.cfg
INFO: platform file exists: /home/pierre/.pyenv/versions/3.9.7/lib/python3.9/site-packages/pythran/pythran-linux.cfg
INFO: sys file exists: /home/pierre/.virtualenvs/env_transonic/lib/python3.8/site-packages/pythran/pythran.cfg
INFO: platform file exists: /home/pierre/.virtualenvs/env_transonic/lib/python3.8/site-packages/pythran/pythran-linux.cfg
INFO: user file exists: /home/pierre/.pythranrc
INFO: pythranrc section [pythran] is valid and options are correct
INFO: pythranrc section [typing] is valid and options are correct
INFO: pythranrc section [compiler] is valid and options are correct
INFO: Generated module: tmp_665062b17d6f48d53bd16f7686fb00c3
INFO: Output: tmp_665062b17d6f48d53bd16f7686fb00c3.tmp
%%%% Output: stream
File /home/pierre/Dev/transonic/doc/ipynb/executed/__pythran__/tmp_665062b17d6f48d53bd16f7686fb00c3.cpython-39-x86_64-linux-gnu.so created by pythran
File /home/pierre/Dev/transonic/doc/ipynb/executed/__pythran__/tmp_665062b17d6f48d53bd16f7686fb00c3.cpython-38-x86_64-linux-gnu.so created by pythran
Pythran stdout:
running build_ext
new_compiler returns <class 'distutils.unixccompiler.UnixCCompiler'>
building 'tmp_665062b17d6f48d53bd16f7686fb00c3' extension
C compiler: clang++-6.0 -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -fPIC
C compiler: clang++-10 -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC
creating /tmp/tmpnp6vi8g7/tmp
compile options: '-DENABLE_PYTHON_MODULE -D__PYTHRAN__=3 -DPYTHRAN_BLAS_BLAS -I/home/pierre/.pyenv/versions/3.9.7/lib/python3.9/site-packages/pythran -I/home/pierre/.pyenv/versions/3.9.7/lib/python3.9/site-packages/pythran/pythonic/patch -I/home/pierre/.pyenv/versions/3.9.7/lib/python3.9/site-packages/numpy/core/include -I/usr/local/include -I/usr/include -I/home/pierre/.pyenv/versions/3.9.7/include -I/home/pierre/.pyenv/versions/3.9.7/include/python3.9 -c'
creating /tmp/tmp62qpahtg/tmp
compile options: '-DENABLE_PYTHON_MODULE -D__PYTHRAN__=3 -DPYTHRAN_BLAS_BLAS -I/home/pierre/.virtualenvs/env_transonic/lib/python3.8/site-packages/pythran -I/home/pierre/.virtualenvs/env_transonic/lib/python3.8/site-packages/pythran/pythonic/patch -I/home/pierre/.virtualenvs/env_transonic/lib/python3.8/site-packages/numpy/core/include -I/usr/local/include -I/usr/include -I/home/pierre/.virtualenvs/env_transonic/include -I/home/pierre/.virtualenvs/env_transonic/include -I/usr/include/python3.8 -c'
extra options: '-std=c++11 -fno-math-errno -fvisibility=hidden -fno-wrapv -Wno-unused-function -Wno-int-in-bool-context -Wno-unknown-warning-option'
clang++-6.0: /tmp/tmp8w2cjl3p.cpp
clang++-6.0 -pthread -shared -L/home/pierre/.pyenv/versions/3.9.7/lib -L/home/pierre/.pyenv/versions/3.9.7/lib /tmp/tmpnp6vi8g7/tmp/tmp8w2cjl3p.o -L/usr/lib/x86_64-linux-gnu -lblas -lblas -o /tmp/tmpsv7pgmcl/tmp_665062b17d6f48d53bd16f7686fb00c3.cpython-39-x86_64-linux-gnu.so -fvisibility=hidden -Wl,-strip-all
clang++-10: /tmp/tmpm48d1zej.cpp
clang++-10 -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fwrapv -O2 -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 /tmp/tmp62qpahtg/tmp/tmpm48d1zej.o -lblas -lblas -o /tmp/tmpgr3vswza/tmp_665062b17d6f48d53bd16f7686fb00c3.cpython-38-x86_64-linux-gnu.so -fvisibility=hidden -Wl,-strip-all
4,
%% Cell type:code id: tags:
``` python
ls --color=never __pythran__
```
%%%% Output: stream
tmp.py
tmp.pythran
tmp_665062b17d6f48d53bd16f7686fb00c3.cpython-39-x86_64-linux-gnu.so
tmp_fdadadf0c3880ac863b8957a4d5dac41.cpython-39-x86_64-linux-gnu.so
tmp_665062b17d6f48d53bd16f7686fb00c3.cpython-38-x86_64-linux-gnu.so
tmp_fdadadf0c3880ac863b8957a4d5dac41.cpython-38-x86_64-linux-gnu.so
__pycache__/
%% Cell type:code id: tags:
``` python
ts_mod = get_ts_mod_display_state("tmp")
print("use the compiled functions:", func(1))
```
%%%% Output: stream
(is_transpiled, is_compiling, is_compiled) = (True, False, True)
use the compiled functions: 4
%% Cell type:code id: tags:
``` python
%run tmp.py
```
%%%% Output: stream
result func(1): 4
%% Cell type:markdown id: tags:
We now update the module such that the Pythran functions are the same as in the first example.
%% Cell type:code id: tags:
``` python
src2 = """
from transonic import boost
@boost
def func(n: int):
return 4 * n
def func_hello():
print("hello")
if __name__ == "__main__":
print("result func(1):", func(1))
"""
with open("tmp.py", "w") as file:
file.write(src2)
```
%% Cell type:code id: tags:
``` python
%run tmp.py
```
%%%% Output: stream
Running transonic on file /home/pierre/Dev/transonic/doc/ipynb/executed/tmp.py... WARNING Code in file /home/pierre/Dev/transonic/doc/ipynb/executed/__pythran__/tmp.py already up-to-date.
Done!
Running transonic on file /home/pierre/Dev/transonic/doc/ipynb/executed/tmp.py... Done!
result func(1): 4
%%%% Output: stream
Code in file /home/pierre/Dev/transonic/doc/ipynb/executed/__pythran__/tmp.py already up-to-date.
%% Cell type:code id: tags:
``` python
ts_mod = get_ts_mod_display_state("tmp")
```
%%%% Output: stream
(is_transpiled, is_compiling, is_compiled) = (True, False, True)
%% Cell type:markdown id: tags:
Great! No need for compilation because the extension has been cached!
%% Cell type:code id: tags:
``` python
while ts_mod.is_compiling:
print(func(1), end=",")
time.sleep(1)
```
%% Cell type:code id: tags: