Commit 43a7baac authored by Ashwin Vishnu's avatar Ashwin Vishnu
Browse files

Merge

......@@ -10,18 +10,17 @@ try: # python 3
except: # python 2.7
from ConfigParser import ConfigParser
sections_libs = ['fftw3', 'fftw3_mpi', 'cufft', 'pfft', 'p3dfft']
def get_default_config():
config = ConfigParser()
# todo: replace by 'fftw3' and 'fftw3_mpi'
sections = ['fftw', 'fftw-mpi', 'cufft', 'pfft', 'p3dfft']
use = {k: 'False' for k in sections}
use['fftw'] = 'True'
use = {k: 'False' for k in sections_libs}
use['fftw3'] = 'True'
for section in sections:
for section in sections_libs:
config.add_section(section)
config.set(section, 'use', use[section])
config.set(section, 'dir', '')
......@@ -64,5 +63,36 @@ def get_config():
return config_dict
def parse_config():
config = get_config()
TMP = os.getenv('FFTW3_INC_DIR')
if TMP is not None:
print('Use value in FFTW3_INC_DIR')
config['fftw3']['include_dir'] = TMP
TMP = os.getenv('FFTW3_LIB_DIR')
if TMP is not None:
print('Use value in FFTW3_LIB_DIR')
config['fftw3']['library_dir'] = TMP
lib_flags_dict = {}
if config['fftw3']['use'] == 'mkl':
lib_flags_dict['fftw3'] = [
'mkl_intel_ilp64', 'mkl_sequential', 'mkl_core']
lib_dirs_dict = {}
for lib in sections_libs:
cfg = config[lib]
if len(cfg['dir']) > 0:
lib_dirs_dict[lib] = os.path.join(cfg['dir'], 'lib')
path = cfg['library_dir']
if len(path) > 0:
lib_dirs_dict[lib] = path
return config, lib_flags_dict, lib_dirs_dict
if __name__ == '__main__':
make_site_cfg_default_file()
......@@ -57,6 +57,9 @@ try:
can_import_mpi4py = True
except ImportError:
can_import_mpi4py = False
else:
if mpi4py.__version__[0] < '2':
raise ValueError('Please upgrade to mpi4py >= 2.0')
short_version = '.'.join([str(i) for i in sys.version_info[:2]])
......@@ -110,9 +113,10 @@ class CompilationError(Exception):
class Extension(object):
def __init__(self, name, sources=None, language=None, ):
def __init__(self, name, sources=None, libraries=None, language=None):
self.name = name
self.sources = sources
self.libraries = libraries
self.language = language
......@@ -179,6 +183,7 @@ class FunctionsRunner(CommandsRunner):
def make_function_cpp_from_pyx(cpp_file, pyx_file,
include_dirs=None,
full_module_name=None, options=None):
path_dir = os.path.split(cpp_file)[0]
if not os.path.exists(path_dir):
......@@ -190,13 +195,9 @@ def make_function_cpp_from_pyx(cpp_file, pyx_file,
if not can_import_cython:
raise ImportError('Can not import Cython.')
include_path = None
if options is not None and 'include_dirs' in options.keys():
include_path = list(options['include_dirs'])
options = CompilationOptions(
cython_default_options,
include_path=include_path,
include_path=include_dirs,
output_file=cpp_file,
cplus=True,
compile_time_env={'MPI4PY': can_import_mpi4py})
......@@ -206,7 +207,8 @@ def make_function_cpp_from_pyx(cpp_file, pyx_file,
{'options': options, 'full_module_name': full_module_name})
def make_command_obj_from_cpp(obj_file, cpp_file, options=None):
def make_command_obj_from_cpp(obj_file, cpp_file, include_dirs=None,
options=None):
path_dir = os.path.split(obj_file)[0]
if not os.path.exists(path_dir):
os.makedirs(path_dir)
......@@ -218,8 +220,8 @@ def make_command_obj_from_cpp(obj_file, cpp_file, options=None):
conf_vars = copy(config_vars)
for k in keys:
print(k, conf_vars[k])
# for k in keys:
# print(k, conf_vars[k])
# problem: it replaces the value (add it better?)
if options is not None:
......@@ -242,19 +244,26 @@ def make_command_obj_from_cpp(obj_file, cpp_file, options=None):
command = [w for w in command.split()
if w not in ['-g', '-DNDEBUG', '-Wstrict-prototypes']]
include_dirs = [conf_vars['INCLUDEPY']]
if can_import_mpi4py and not cpp_file.endswith('.cu'):
command[0] = os.getenv('MPICXX', 'mpicxx')
if 'cufft' in cpp_file:
include_dirs.extend([
'/opt/cuda/NVIDIA_CUDA-6.0_Samples/common/inc/'])
includepy = [conf_vars['INCLUDEPY']]
if cpp_file.endswith('.cu'):
include_dirs.extend([
'/usr/lib/openmpi/include',
'/usr/lib/openmpi/include/openmpi'])
if include_dirs is None:
include_dirs = includepy
else:
include_dirs = includepy + include_dirs
if options is not None and 'include_dirs' in options.keys():
include_dirs.extend(options['include_dirs'])
# this should not be here! No hard-coded path...
# if 'cufft' in cpp_file:
# include_dirs.extend([
# '/opt/cuda/NVIDIA_CUDA-6.0_Samples/common/inc/'])
# cu + mpi ? No hard-coded path...
# if cpp_file.endswith('.cu'):
# include_dirs.extend([
# '/usr/lib/openmpi/include',
# '/usr/lib/openmpi/include/openmpi'])
command += ['-I' + incdir for incdir in include_dirs]
command += ['-c', cpp_file, '-o', obj_file]
......@@ -262,8 +271,8 @@ def make_command_obj_from_cpp(obj_file, cpp_file, options=None):
def make_command_ext_from_objs(
ext_file, obj_files, lib_dirs=None,
libraries=None, options=None):
ext_file, obj_files, lib_flags=None, lib_dirs=None,
options=None):
if not has_to_build(ext_file, obj_files):
return
......@@ -290,13 +299,15 @@ def make_command_ext_from_objs(
if lib_dirs is not None:
command.extend(['-L' + lib_dir for lib_dir in lib_dirs])
if libraries is not None:
command.extend(['-l' + lib for lib in libraries])
if lib_flags is not None:
command.extend(['-l' + lib for lib in lib_flags])
return command
def make_extensions(extensions, lib_dirs=None, libraries=None,
def make_extensions(extensions,
include_dirs=None,
lib_dirs_dict=None, lib_flags_dict=None,
**options):
if all(command not in sys.argv for command in [
......@@ -339,7 +350,9 @@ def make_extensions(extensions, lib_dirs=None, libraries=None,
command = make_function_cpp_from_pyx(
cpp_file, pyx_file,
full_module_name=full_module_name, options=options)
full_module_name=full_module_name,
include_dirs=include_dirs,
options=options)
if command is not None:
commands.append(command)
......@@ -351,7 +364,8 @@ def make_extensions(extensions, lib_dirs=None, libraries=None,
commands = []
for path in files['cpp']:
result = os.path.join(path_tmp, os.path.splitext(path)[0] + '.o')
command = make_command_obj_from_cpp(result, path, options)
command = make_command_obj_from_cpp(
result, path, include_dirs, options)
if command is not None:
commands.append(command)
files['o'].append(result)
......@@ -368,8 +382,19 @@ def make_extensions(extensions, lib_dirs=None, libraries=None,
objects = [
os.path.join(path_tmp, os.path.splitext(source)[0] + '.o')
for source in ext.sources]
lib_dirs = []
lib_flags = []
for lib in ext.libraries:
if lib_dirs_dict is not None and lib in lib_dirs_dict:
lib_dirs.append(lib_dirs_dict[lib])
if lib_flags_dict is not None and lib in lib_flags_dict:
lib_flags.extend(lib_flags_dict[lib])
else:
lib_flags.append(lib)
command = make_command_ext_from_objs(
result, objects, lib_dirs=lib_dirs, libraries=libraries)
result, objects, lib_dirs=lib_dirs, lib_flags=lib_flags)
if command is not None:
commands.append(command)
files['so'].append(result)
......
......@@ -24,7 +24,7 @@ Distribution(dict(setup_requires=setup_requires))
from numpy.distutils.system_info import get_info
from purepymake import Extension, make_extensions
from config import get_config
from config import parse_config
from src_cy.make_files_with_mako import make_pyx_files
try:
......@@ -40,7 +40,7 @@ except KeyError:
make_pyx_files()
config = get_config()
config, lib_flags_dict, lib_dirs_dict = parse_config()
# Get the long description from the relevant file
with open('README.rst') as f:
......@@ -106,18 +106,30 @@ def create_ext(base_name):
[os.path.join(src_base, 'base_fftmpi.cpp'),
os.path.join(src_cpp_dim, 'base_fft' + dim + 'mpi.cpp')])
libraries = ['fftw3']
if 'fftwmpi' in base_name:
libraries.append('fftw3_mpi')
elif 'pfft' in base_name:
libraries.extend(['fftw3_mpi', 'pfft'])
elif 'p3dfft' in base_name:
libraries.append('p3dfft')
elif 'cufft' in base_name:
libraries.append(['cufft', 'mpi_cxx'])
return Extension(
name='fluidfft.fft' + dim + '.' + base_name,
sources=sources)
sources=sources,
libraries=libraries)
base_names = []
if config['fftw']['use']:
if config['fftw3']['use']:
base_names.extend([
'fft2d_with_fftw1d', 'fft2d_with_fftw2d', 'fft2dmpi_with_fftw1d',
'fft3d_with_fftw3d', 'fft3dmpi_with_fftw1d'])
if config['fftw-mpi']['use']:
if config['fftw3_mpi']['use']:
if use_mkl_intel_lp64:
warn('When numpy uses mkl (as for example with conda), '
'there are symbol conflicts between mkl and fftw. '
......@@ -131,13 +143,12 @@ if config['cufft']['use']:
base_names.extend(['fft2d_with_cufft'])
base_names.extend(['fft3d_with_cufft'])
if config['pfft']['use']:
if config['pfft']['use'] and not use_mkl_intel_lp64:
base_names.extend(['fft3dmpi_with_pfft'])
if config['p3dfft']['use']:
base_names.extend(['fft3dmpi_with_p3dfft'])
# todo: include_dirs has to be a list (ordered)
on_rtd = os.environ.get('READTHEDOCS')
if on_rtd:
......@@ -146,16 +157,9 @@ else:
import numpy as np
ext_modules = []
# todo: this has to be in config ("library_flags")
if config['fftw']['use'] == 'mkl':
libraries = set(('mkl_intel_ilp64', 'mkl_sequential', 'mkl_core'))
else:
libraries = set(['fftw3'])
lib_dirs = set()
include_dirs = set(
[src_cy_dir, src_base, src_cpp_3d, src_cpp_2d,
'include', np.get_include()])
include_dirs = [
src_cy_dir, src_base, src_cpp_3d, src_cpp_2d,
'include', np.get_include()]
try:
import mpi4py
......@@ -163,54 +167,36 @@ else:
warn('ImportError for mpi4py: '
"all extensions based on mpi won't be built.")
base_names = [name for name in base_names if 'mpi' not in name]
CXX = None
else:
CXX = os.getenv('MPICXX', 'mpicxx')
if mpi4py.__version__[0] < '2':
raise ValueError('Please upgrade to mpi4py >= 2.0')
include_dirs.add(mpi4py.get_include())
include_dirs.append(mpi4py.get_include())
# todo: libraries have to be attached to extensions
# todo: library paths have to be handle in config
def update_with_config(key):
cfg = config[key]
if len(cfg['dir']) > 0:
path = cfg['dir']
include_dirs.add(os.path.join(path, 'include'))
lib_dirs.add(os.path.join(path, 'lib'))
path = os.path.join(cfg['dir'], 'include')
if path not in include_dirs:
include_dirs.append(path)
if len(cfg['include_dir']) > 0:
include_dirs.add(cfg['include_dir'])
if len(cfg['library_dir']) > 0:
lib_dirs.add(cfg['library_dir'])
path = cfg['include_dir']
if path not in include_dirs:
include_dirs.append(path)
if config['fftw3']['use']:
update_with_config('fftw3')
if config['fftw']['use']:
update_with_config('fftw')
keys = ['pfft', 'p3dfft', 'cufft']
for base_name in base_names:
ext_modules.append(create_ext(base_name))
# todo: this has to be in config
TMP = os.getenv('FFTW3_INC_DIR')
if TMP is not None:
include_dirs.add(TMP)
TMP = os.getenv('FFTW3_LIB_DIR')
if TMP is not None:
lib_dirs.add(TMP)
if 'fftwmpi' in base_name:
libraries.add('fftw3_mpi')
update_with_config('fftw-mpi')
elif 'pfft' in base_name:
libraries.update(['fftw3_mpi', 'pfft'])
update_with_config('pfft')
elif 'p3dfft' in base_name:
libraries.add('p3dfft')
update_with_config('p3dfft')
elif 'cufft' in base_name:
libraries.update(['cufft', 'mpi_cxx'])
update_with_config('cufft')
update_with_config('fftw3_mpi')
for key in keys:
if key in base_name:
update_with_config(key)
# todo: pythran build has to be in purepymake (in parallel)
......@@ -245,8 +231,8 @@ def make_pythran_extensions(modules):
if not on_rtd:
make_extensions(
ext_modules, include_dirs=include_dirs,
lib_dirs=lib_dirs, libraries=libraries,
CXX=CXX, CFLAGS='-std=c++03')
lib_flags_dict=lib_flags_dict, lib_dirs_dict=lib_dirs_dict,
CFLAGS='-std=c++03')
# Clear all purepymake.Extension objects after build is done.
ext_modules = []
......@@ -275,7 +261,7 @@ setup(
# 5 - Production/Stable
'Development Status :: 4 - Beta',
'Intended Audience :: Science/Research',
'Intended Audience :: Education',#
'Intended Audience :: Education',
'Topic :: Scientific/Engineering',
'License :: OSI Approved :: GNU General Public License v2 (GPLv2)',
# actually CeCILL License (GPL compatible license for French laws)
......
[fftw]
use = True
[fftw3]
use = False
dir =
include_dir =
library_dir =
[fftw-mpi]
[fftw3_mpi]
use = False
dir =
include_dir =
......
[fftw]
[fftw3]
use = True
dir =
include_dir =
library_dir =
[fftw-mpi]
[fftw3_mpi]
use = True
dir =
include_dir =
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment