Commit 5f6d8153 authored by Ashwin Vishnu's avatar Ashwin Vishnu
Browse files

Consolidate compiler configurations into get_distconfig

Also,
1. Provides a way to build MPI Cython extensions which uses sequential fftw
   library.
2. Should prevent KeyError if variable is missing in distutils.sysconfig.
   Fixes #29
parent ad4f3d9abc1a
Pipeline #15106 passed with stage
in 5 minutes and 19 seconds
import os
import sys
from concurrent.futures import ThreadPoolExecutor as Pool
from runpy import run_path
from pathlib import Path
from runpy import run_path
from warnings import warn
from distutils import sysconfig as dsysconfig
from setuptools.command.build_ext import build_ext
from setup_make import make_extensions, make_pythran_extensions
from setup_configure import (
configuration,
lib_flags_dict,
lib_dirs_dict,
lib_flags_dict,
num_procs,
)
from setup_make import make_extensions, make_pythran_extensions
config_vars = dsysconfig.get_config_vars()
here = Path(__file__).parent.absolute()
try:
# pythran > 0.8.6
from pythran.dist import PythranExtension, PythranBuildExt
from pythran.dist import PythranBuildExt, PythranExtension
can_import_pythran = True
except ImportError:
......
......@@ -2,10 +2,12 @@
site.cfg.default to site.cfg and modify this file.
"""
import multiprocessing
import os
import sys
import sysconfig
from configparser import ConfigParser
import multiprocessing
from distutils import sysconfig as dsysconfig
from distutils.util import strtobool
sections_libs = ["fftw3", "fftw3_mpi", "cufft", "pfft", "p3dfft"]
......@@ -192,10 +194,54 @@ def parse_config():
return config, lib_flags_dict, lib_dirs_dict
def get_distconfig():
"""Get configuration for building extensions -- collected from
distutils.sysconfig, sysconfig, and fluidfft's site.cfg.
"""
global configuration # site.cfg
site_cfg = configuration["environ"]
distutil_sysconfig = dsysconfig.get_config_vars()
short_version = sysconfig.get_python_version()
platform_pyversion = "-".join([sysconfig.get_platform(), short_version])
def site_dist_default(key, default):
return site_cfg.get(key, distutil_sysconfig.get(key, default))
distconfig = dict(
# PATH_LIB_PYTHON=os.path.join(
# sys.prefix, "lib", "python" + short_version, "site-packages"
# ),
PATH_TMP="build/temp." + platform_pyversion,
PATH_LIB="build/lib." + platform_pyversion,
EXT_SUFFIX=sysconfig.get_config_var("EXT_SUFFIX") or ".so",
MPICXX=site_cfg.get("MPICXX", "mpicxx"),
CXX=site_dist_default("CXX", "cxx"),
LDSHARED=site_dist_default("LDSHARED", "cxx -shared"),
CFLAGS=site_dist_default("CFLAGS", ""),
INCLUDEPY=distutil_sysconfig.get(
"INCLUDEPY",
# Again! For Pypy (see: https://foss.heptapod.net/pypy/pypy/issues/2478)
sysconfig.get_config_var("INCLUDEPY"),
),
)
# make_command_obj_from_cpp
for key in ("OPT", "BASECFLAGS", "CCSHARED"):
distconfig[key] = distutil_sysconfig.get(key, "")
return distconfig
configuration, lib_flags_dict, lib_dirs_dict = parse_config()
libs_mpi = ["fftw3_mpi", "pfft", "p3dfft"]
build_needs_mpi4py = any([configuration[lib]["use"] for lib in libs_mpi])
# The site.cfg MPICXX acts as an override to properly compile extensions like
# fft2d.mpi_with_fftw1d and fft3d.mpi_with_fftw1d when MPI and fftw libraries
# are available but fftw3-mpi library is not.
build_needs_mpi4py = any([configuration[lib]["use"] for lib in libs_mpi]) or configuration["environ"].get("MPICXX", False)
if DISABLE_MPI:
build_needs_mpi4py = False
......
......@@ -32,8 +32,6 @@ import sys
from time import sleep
import os
from datetime import datetime
import sysconfig
from distutils import sysconfig as dsysconfig
import subprocess
from copy import copy
import multiprocessing
......@@ -42,10 +40,10 @@ from pathlib import Path
from setuptools import Extension as SetuptoolsExtension
from setup_configure import build_needs_mpi4py, num_procs, TRANSONIC_BACKEND
from setup_configure import build_needs_mpi4py, get_distconfig, num_procs, TRANSONIC_BACKEND
config_vars = dsysconfig.get_config_vars()
distconfig = get_distconfig()
here = Path(__file__).parent.absolute()
......@@ -73,17 +71,6 @@ if build_needs_mpi4py:
)
short_version = sysconfig.get_python_version()
platform_pyversion = "-".join([sysconfig.get_platform(), short_version])
path_lib_python = os.path.join(
sys.prefix, "lib", "python" + short_version, "site-packages"
)
path_tmp = "build/temp." + platform_pyversion
path_lib = "build/lib." + platform_pyversion
def modification_date(filename):
t = os.path.getmtime(filename)
return datetime.fromtimestamp(t)
......@@ -251,24 +238,13 @@ def make_command_obj_from_cpp(
if not has_to_build(obj_file, (cpp_file,)):
return
keys = ["CXX", "OPT", "BASECFLAGS", "CFLAGS", "CCSHARED"]
conf_vars = copy(config_vars)
# for k in keys:
# print(k, conf_vars[k])
_distconfig = copy(distconfig)
# problem: it replaces the value (add it better?)
if options is not None:
for k, v in options.items():
if v is not None:
conf_vars[k] = v
# for Pypy
if "BASECFLAGS" not in conf_vars:
keys.remove("BASECFLAGS")
_distconfig.update({k: v for k, v in options.items() if v is not None})
command = " ".join([conf_vars[k] for k in keys])
keys = ["CXX", "OPT", "BASECFLAGS", "CFLAGS", "CCSHARED"]
command = " ".join([_distconfig[k] for k in keys])
if cpp_file.endswith(".cu"):
NVCC = os.getenv("NVCC", "nvcc")
......@@ -294,15 +270,11 @@ def make_command_obj_from_cpp(
continue
command.append(word)
else:
mpicxx = os.getenv("MPICXX", "mpicxx").split()
mpicxx = _distconfig["MPICXX"].split()
mpicxx.extend(command[1:])
command = mpicxx
try:
includepy = conf_vars["INCLUDEPY"]
except KeyError:
# Again! For Pypy (see: https://foss.heptapod.net/pypy/pypy/issues/2478)
includepy = sysconfig.get_config_var("INCLUDEPY")
includepy = _distconfig["INCLUDEPY"]
includedir = os.path.split(includepy)[0]
if os.path.split(includedir)[-1] == "include":
......@@ -330,8 +302,8 @@ def make_command_ext_from_objs(
if not os.path.exists(path_dir):
os.makedirs(path_dir)
cxx = config_vars["CXX"]
ldshared = os.getenv("LDSHARED", config_vars["LDSHARED"])
cxx = distconfig["CXX"]
ldshared = distconfig["LDSHARED"]
if "mpi_with_p3dfft" in ext_file or "mpi_with_pfft" in ext_file:
# remove an option incompatible with these libraries
......@@ -378,7 +350,7 @@ def make_extensions(
):
return []
path_base_output = path_lib
path_base_output = distconfig["PATH_LIB"]
sources = set()
for ext in extensions:
......@@ -446,7 +418,7 @@ def make_extensions(
# compile .cpp files if needed
commands = []
for path in files["cpp"]:
result = os.path.join(path_tmp, os.path.splitext(path)[0] + ".o")
result = os.path.join(distconfig["PATH_TMP"], os.path.splitext(path)[0] + ".o")
command = make_command_obj_from_cpp(result, path, include_dirs, options)
if command is not None:
commands.append(command)
......@@ -460,12 +432,12 @@ def make_extensions(
files["so"] = []
commands = []
for ext in extensions:
suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so"
suffix = distconfig["EXT_SUFFIX"]
result = os.path.join(
path_base_output, ext.name.replace(".", os.path.sep) + suffix
)
objects = [
os.path.join(path_tmp, os.path.splitext(source)[0] + ".o")
os.path.join(distconfig["PATH_TMP"], os.path.splitext(source)[0] + ".o")
for source in ext.sources
]
......@@ -516,7 +488,7 @@ def make_pythran_extensions():
base_file = mod.replace(".", os.path.sep)
py_file = base_file + ".py"
# warning: does not work on Windows (?)
suffix = sysconfig.get_config_var("EXT_SUFFIX") or ".so"
suffix = distconfig["EXT_SUFFIX"]
bin_file = base_file + suffix
if (
not develop
......
......@@ -33,6 +33,8 @@ library_dir =
## extensions.
## To modify compiler used to build Cython extensions:
# CXX =
# CFLAGS =
# MPICXX =
# LDSHARED =
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment