Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Gloo python bindings #585

Merged
merged 17 commits into from
May 20, 2022
Merged

Gloo python bindings #585

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
fd293c5
fixing standalone gloo communicator
nirandaperera May 16, 2022
cb50036
fixing gloo cython build
nirandaperera May 18, 2022
b113ace
minor change
nirandaperera May 16, 2022
5b6d48c
minor change
nirandaperera May 18, 2022
5f22623
minor change
nirandaperera May 18, 2022
45dd8e9
fixing cython
nirandaperera May 18, 2022
6147c9d
enabling macos tests
nirandaperera May 18, 2022
0398304
fixing cython build
nirandaperera May 18, 2022
dccab18
attempting to fix cython
nirandaperera May 19, 2022
d9fbff2
attempting to fix macos and windows
nirandaperera May 19, 2022
94e3421
fix
nirandaperera May 19, 2022
2d73794
minor change
nirandaperera May 19, 2022
2df750c
minor change
nirandaperera May 19, 2022
2a744b3
bumping macos python to 3.9
nirandaperera May 19, 2022
9338a0a
removing ipath from macos
nirandaperera May 19, 2022
e172fbf
adding fsspec to macos yml
nirandaperera May 19, 2022
e1cc3ce
changing status
nirandaperera May 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,11 +270,13 @@ def build_python():
elif os.name == 'nt':
env["ARROW_PREFIX"] = str(Path(os.environ["CONDA_PREFIX"], "Library"))

env['CYLON_GLOO'] = str(CYLON_GLOO)
env['GLOO_PREFIX'] = GLOO_PREFIX
env['CYLON_UCX'] = str(CYLON_UCX)
env['CYLON_UCC'] = str(CYLON_UCC)
env['UCC_PREFIX'] = UCC_PREFIX
if CYLON_GLOO:
env['CYLON_GLOO'] = str(CYLON_GLOO)
env['GLOO_PREFIX'] = GLOO_PREFIX
if CYLON_UCC and CYLON_UCX:
env['CYLON_UCX'] = str(CYLON_UCX)
env['CYLON_UCC'] = str(CYLON_UCC)
env['UCC_PREFIX'] = UCC_PREFIX

logger.info("Arrow prefix: " + str(Path(os.environ["CONDA_PREFIX"])))
res = subprocess.run(python_build_command, shell=True, env=env, cwd=PYTHON_SOURCE_DIR)
Expand Down
45 changes: 36 additions & 9 deletions python/pycylon/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,34 @@

compiler_directives = {"language_level": 3, "embedsignature": True}

cython_files = ["pycylon/*/*.pyx"]
print("CYTHON: " + str(cython_files))
cython_files = ['pycylon/net/channel.pyx',
'pycylon/net/comm_type.pyx',
'pycylon/net/communicator.pyx',
'pycylon/net/comm_config.pyx',
'pycylon/net/mpi_config.pyx',
'pycylon/net/txrequest.pyx',
'pycylon/net/comm_ops.pyx',
'pycylon/_libs/index.pyx',
'pycylon/data/scalar.pyx',
'pycylon/data/groupby.pyx',
'pycylon/data/arrow_util.pyx',
'pycylon/data/aggregates.pyx',
'pycylon/data/column.pyx',
'pycylon/data/csv.pyx',
'pycylon/data/compute.pyx',
'pycylon/data/table.pyx',
'pycylon/data/data_type.pyx',
'pycylon/api/lib.pyx',
'pycylon/api/types.pyx',
'pycylon/indexing/cyindex.pyx',
'pycylon/indexing/index_utils.pyx',
'pycylon/ctx/context.pyx',
'pycylon/io/csv_write_config.pyx',
'pycylon/io/csv_read_config.pyx',
'pycylon/common/status.pyx',
'pycylon/common/join_config.pyx',
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This looks okay. We can modularize it later on.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

pycylon/data/**, pycylon/api./** something like this, but I am not sure, what is the best way to capture the modularized ones. But the existing one is satisfactory for the moment.

'pycylon/common/code.pyx',
'pycylon/util/logging.pyx']

if not CYLON_PREFIX:
raise ValueError("CYLON_PREFIX not set")
Expand Down Expand Up @@ -135,9 +161,10 @@
mpi_include_dir = [mpi4py.get_config()['include_dirs']]
_include_dirs.extend(mpi_include_dir)


macros = []
if CYLON_GLOO:
cython_files.append('pycylon/net/gloo_config.pyx')

libraries.append('gloo')
library_directories.append(os.path.join(GLOO_PREFIX, 'lib'))
_include_dirs.append(os.path.join(GLOO_PREFIX, 'include'))
Expand All @@ -151,11 +178,11 @@
macros.append(('BUILD_CYLON_UCX', '1'))
macros.append(('BUILD_CYLON_UCC', '1'))


print('Libraries:', libraries)
print("Lib dirs:", library_directories)
print("Include dirs:", _include_dirs)
print("Macros:", macros)
print("CYTHON files : " + str(cython_files))
print('Libraries :', libraries)
print("Lib dirs :", library_directories)
print("Include dirs :", _include_dirs)
print("Macros :", macros)

# Adopted the Cudf Python Build format
# https://github.com/rapidsai/cudf
Expand All @@ -170,7 +197,7 @@
extra_link_args=extra_link_args,
libraries=libraries,
library_dirs=library_directories,
define_macros=macros
define_macros=macros,
)
]

Expand Down