Skip to content

Commit cd0cfc0

Browse files
multiphaseCFDmultiphaseCFD
authored
Add LGPU-MPI cpp backend (#514)
* add LGPU-MPI cpp layer
1 parent bd9af0f commit cd0cfc0

File tree

73 files changed

+11411
-61
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

73 files changed

+11411
-61
lines changed

‎.github/CHANGELOG.md

+3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
### New features since last release
44

5+
* Integrate the distributed C++ backend of Lightning-GPU into the Lightning monorepo.
6+
[(#514)] (https://github.com/PennyLaneAI/pennylane-lightning/pull/514)
7+
58
* Integrate Lightning-GPU into the Lightning monorepo. The new backend is named `lightning_gpu` and includes all single-GPU features.
69
[(#499)] (https://github.com/PennyLaneAI/pennylane-lightning/pull/499)
710

‎.github/workflows/tests_gpu_cu11.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Testing::Linux::x86_64 (Lightning-GPU)
1+
name: Testing::Linux::x86_64::LGPU
22
on:
33
pull_request:
44
push:
@@ -143,7 +143,7 @@ jobs:
143143
if-no-files-found: error
144144

145145
pythontestswithLGPU:
146-
needs: [builddeps]
146+
needs: [builddeps, cpptestswithLGPU_cu11]
147147
strategy:
148148
matrix:
149149
os: [ubuntu-22.04]

‎.github/workflows/tests_linux_x86_mpi.yml

+185
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
name: Tests::Linux::x86_64::LGPU::MPI
2+
on:
3+
workflow_call:
4+
inputs:
5+
lightning-version:
6+
type: string
7+
required: true
8+
description: The version of lightning to use. Valid values are either 'stable' (most recent git-tag) or 'latest' (most recent commit from master)
9+
pennylane-version:
10+
type: string
11+
required: true
12+
description: The version of PennyLane to use. Valid values are either 'stable' (most recent git-tag) or 'latest' (most recent commit from master)
13+
release:
14+
push:
15+
branches:
16+
- main
17+
pull_request:
18+
19+
env:
20+
COVERAGE_FLAGS: "--cov=pennylane_lightning --cov-report=term-missing --cov-report=xml:./coverage.xml --no-flaky-report -p no:warnings --tb=native"
21+
GCC_VERSION: 11
22+
OMP_NUM_THREADS: "2"
23+
CI_CUDA_ARCH: 86
24+
25+
concurrency:
26+
group: gpu-test-mpi-${{ github.ref }}-${{ inputs.lightning-version }}-${{ inputs.pennylane-version }}
27+
cancel-in-progress: true
28+
29+
jobs:
30+
cpp_tests:
31+
if: contains(github.event.pull_request.labels.*.name, 'ci:use-multi-gpu-runner') || (inputs.lightning-version != '' && inputs.pennylane-version != '')
32+
runs-on:
33+
- self-hosted
34+
- linux
35+
- x64
36+
- ubuntu-22.04
37+
- multi-gpu
38+
strategy:
39+
max-parallel: 1
40+
matrix:
41+
mpilib: ["mpich", "openmpi"]
42+
timeout-minutes: 30
43+
44+
steps:
45+
- name: Checkout
46+
uses: actions/checkout@v3
47+
with:
48+
fetch-tags: true
49+
50+
- name: Switch to stable build of Lightning-GPU
51+
if: inputs.lightning-version == 'stable'
52+
run: |
53+
git fetch tags --force
54+
git checkout $(git tag | sort -V | tail -1)
55+
- uses: actions/setup-python@v4
56+
id: setup_python
57+
name: Install Python
58+
with:
59+
python-version: '3.9'
60+
61+
# Since the self-hosted runner can be re-used. It is best to set up all package
62+
# installations in a virtual environment that gets cleaned at the end of each workflow run
63+
- name: Setup Python virtual environment
64+
id: setup_venv
65+
env:
66+
VENV_NAME: ${{ github.workspace }}/venv_${{ steps.setup_python.outputs.python-version }}_${{ github.sha }}
67+
run: |
68+
# Clear any pre-existing venvs
69+
rm -rf venv_*
70+
71+
# Create new venv for this workflow_run
72+
python --version
73+
python -m venv ${{ env.VENV_NAME }}
74+
75+
# Add the venv to PATH for subsequent steps
76+
echo ${{ env.VENV_NAME }}/bin >> $GITHUB_PATH
77+
78+
# Adding venv name as an output for subsequent steps to reference if needed
79+
echo "venv_name=${{ env.VENV_NAME }}" >> $GITHUB_OUTPUT
80+
81+
- name: Display Python-Path
82+
id: python_path
83+
run: |
84+
py_path=$(which python)
85+
echo "Python Interpreter Path => $py_path"
86+
echo "python=$py_path" >> $GITHUB_OUTPUT
87+
88+
pip_path=$(which python)
89+
echo "PIP Path => $pip_path"
90+
echo "pip=$pip_path" >> $GITHUB_OUTPUT
91+
92+
- name: Install required packages
93+
run: |
94+
python -m pip install ninja cmake custatevec-cu11
95+
96+
- name: Validate GPU version and installed compiler
97+
run: |
98+
source /etc/profile.d/modules.sh && module use /opt/modules && module load cuda/11.8
99+
which -a nvcc
100+
nvcc --version
101+
- name: Validate Multi-GPU packages
102+
run: |
103+
source /etc/profile.d/modules.sh && module use /opt/modules/ && module load ${{ matrix.mpilib }}
104+
echo 'Checking for ${{ matrix.mpilib }}'
105+
which -a mpirun
106+
mpirun --version
107+
which -a mpicxx
108+
mpicxx --version
109+
module unload ${{ matrix.mpilib }}
110+
- name: Install Latest PennyLane
111+
if: inputs.pennylane-version == 'latest'
112+
run: python -m pip install git+https://github.com/PennyLaneAI/pennylane.git@master
113+
114+
- name: Build and run unit tests
115+
run: |
116+
source /etc/profile.d/modules.sh && module use /opt/modules/ && module load ${{ matrix.mpilib }}
117+
export CUQUANTUM_SDK=$(python -c "import site; print( f'{site.getsitepackages()[0]}/cuquantum/lib')")
118+
cmake . -BBuild \
119+
-DPL_BACKEND=lightning_gpu \
120+
-DENABLE_MPI=ON \
121+
-DCMAKE_BUILD_TYPE=Debug \
122+
-DENABLE_COVERAGE=ON \
123+
-DBUILD_TESTS=ON \
124+
-DCMAKE_CXX_COMPILER=mpicxx \
125+
-DCMAKE_CUDA_COMPILER="/usr/local/cuda/bin/nvcc" \
126+
-DCMAKE_CUDA_ARCHITECTURES="86" \
127+
-DPython_EXECUTABLE:FILE="${{ steps.python_path.outputs.python }}" \
128+
-G Ninja
129+
cmake --build ./Build
130+
cd ./Build
131+
mkdir -p ./tests/results
132+
for file in *runner ; do ./$file --order lex --reporter junit --out ./tests/results/report_$file.xml; done;
133+
for file in *runner_mpi ; do /opt/mpi/${{ matrix.mpilib }}/bin/mpirun -np 2 ./$file --order lex --reporter junit --out ./tests/results/report_$file.xml; done;
134+
lcov --directory . -b ../pennylane_lightning/src --capture --output-file coverage.info
135+
lcov --remove coverage.info '/usr/*' --output-file coverage.info
136+
mv coverage.info coverage-${{ github.job }}-lightning_gpu_${{ matrix.mpilib }}.info
137+
138+
- name: Upload test results
139+
uses: actions/upload-artifact@v3
140+
if: always()
141+
with:
142+
if-no-files-found: error
143+
name: ubuntu-tests-reports
144+
path: ./Build/tests/results/
145+
146+
- name: Upload code coverage results
147+
uses: actions/upload-artifact@v3
148+
with:
149+
if-no-files-found: error
150+
name: ubuntu-codecov-results-cpp
151+
path: ./Build/coverage-${{ github.job }}-lightning_gpu_${{ matrix.mpilib }}.info
152+
153+
- name: Cleanup
154+
if: always()
155+
run: |
156+
rm -rf ${{ steps.setup_venv.outputs.venv_name }}
157+
rm -rf * .git .gitignore .github
158+
pip cache purge
159+
160+
upload-to-codecov-linux-cpp:
161+
needs: ["cpp_tests"]
162+
name: Upload coverage data to codecov
163+
runs-on: ubuntu-latest
164+
steps:
165+
- name: Checkout
166+
uses: actions/checkout@v3
167+
168+
- name: Download coverage reports
169+
uses: actions/download-artifact@v3
170+
with:
171+
name: ubuntu-codecov-results-cpp
172+
173+
- name: Upload to Codecov
174+
uses: codecov/codecov-action@v3
175+
with:
176+
fail_ci_if_error: true
177+
verbose: true
178+
token: ${{ secrets.CODECOV_TOKEN }}
179+
180+
- name: Cleanup
181+
if: always()
182+
run: |
183+
rm -rf ${{ steps.setup_venv.outputs.venv_name }}
184+
rm -rf * .git .gitignore .github
185+
pip cache purge

‎Makefile

+26-26
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,24 @@ TESTRUNNER := -m pytest tests --tb=short
66

77
PL_BACKEND?="$(if $(backend:-=),$(backend),lightning_qubit)"
88

9-
ifdef verbose
10-
VERBOSE := --verbose
11-
else
12-
VERBOSE :=
13-
endif
14-
159
ifdef check
1610
CHECK := --check
1711
else
1812
CHECK :=
1913
endif
2014

15+
ifdef build_options
16+
OPTIONS := $(build_options)
17+
else
18+
OPTIONS :=
19+
endif
20+
21+
ifdef verbose
22+
VERBOSE := --verbose
23+
else
24+
VERBOSE :=
25+
endif
26+
2127
.PHONY: help
2228
help:
2329
@echo "Please use \`make <target>' where <target> is one of"
@@ -56,16 +62,7 @@ clean:
5662
rm -rf .coverage coverage_html_report/
5763
rm -rf pennylane_lightning/*_ops*
5864

59-
.PHONY : test-builtin test-suite test-python coverage coverage-cpp test-cpp test-cpp-no-omp test-cpp-blas test-cpp-kokkos
60-
test-builtin:
61-
PL_DEVICE=$(if $(device:-=),$(device),lightning.qubit) $(PYTHON) -I $(TESTRUNNER)
62-
63-
test-suite:
64-
pl-device-test --device $(if $(device:-=),$(device),lightning.qubit) --skip-ops --shots=20000
65-
pl-device-test --device $(if $(device:-=),$(device),lightning.qubit) --shots=None --skip-ops
66-
67-
test-python: test-builtin test-suite
68-
65+
.PHONY : test-builtin test-suite test-python test-cpp coverage coverage-cpp
6966
coverage:
7067
@echo "Generating coverage report for $(if $(device:-=),$(device),lightning.qubit) device:"
7168
$(PYTHON) $(TESTRUNNER) $(COVERAGE)
@@ -75,20 +72,29 @@ coverage:
7572
coverage-cpp:
7673
@echo "Generating cpp coverage report in BuildCov/out for $(PL_BACKEND) backend"
7774
rm -rf ./BuildCov
78-
cmake -BBuildCov -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DENABLE_COVERAGE=ON -DPL_BACKEND=$(PL_BACKEND)
75+
cmake -BBuildCov -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DENABLE_COVERAGE=ON -DPL_BACKEND=$(PL_BACKEND) $(OPTIONS)
7976
cmake --build ./BuildCov
8077
cd ./BuildCov; for file in *runner ; do ./$file; done; \
8178
lcov --directory . -b ../pennylane_lightning/core/src --capture --output-file coverage.info; \
8279
genhtml coverage.info --output-directory out
8380

8481
build:
8582
rm -rf ./Build
86-
cmake -BBuild -G Ninja -DENABLE_BLAS=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND)
83+
cmake -BBuild -G Ninja -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND) $(OPTIONS)
8784
cmake --build ./Build $(VERBOSE)
8885

86+
test-builtin:
87+
PL_DEVICE=$(if $(device:-=),$(device),lightning.qubit) $(PYTHON) -I $(TESTRUNNER)
88+
89+
test-suite:
90+
pl-device-test --device $(if $(device:-=),$(device),lightning.qubit) --skip-ops --shots=20000
91+
pl-device-test --device $(if $(device:-=),$(device),lightning.qubit) --shots=None --skip-ops
92+
93+
test-python: test-builtin test-suite
94+
8995
test-cpp:
9096
rm -rf ./BuildTests
91-
cmake -BBuildTests -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DENABLE_OPENMP=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND)
97+
cmake -BBuildTests -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTS=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND) $(OPTIONS)
9298
ifdef target
9399
cmake --build ./BuildTests $(VERBOSE) --target $(target)
94100
OMP_PROC_BIND=false ./BuildTests/$(target)
@@ -97,12 +103,6 @@ else
97103
OMP_PROC_BIND=false cmake --build ./BuildTests $(VERBOSE) --target test
98104
endif
99105

100-
test-cpp-blas:
101-
rm -rf ./BuildTests
102-
cmake -BBuildTests -G Ninja -DBUILD_TESTS=ON -DENABLE_BLAS=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND)
103-
cmake --build ./BuildTests $(VERBOSE)
104-
cmake --build ./BuildTests $(VERBOSE) --target test
105-
106106
.PHONY: format format-cpp
107107
format: format-cpp format-python
108108

@@ -115,7 +115,7 @@ format-python:
115115
.PHONY: check-tidy
116116
check-tidy:
117117
rm -rf ./BuildTidy
118-
cmake -BBuildTidy -DENABLE_CLANG_TIDY=ON -DBUILD_TESTS=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND)
118+
cmake -BBuildTidy -DENABLE_CLANG_TIDY=ON -DBUILD_TESTS=ON -DENABLE_WARNINGS=ON -DPL_BACKEND=$(PL_BACKEND) $(OPTIONS)
119119
ifdef target
120120
cmake --build ./BuildTidy $(VERBOSE) --target $(target)
121121
else

‎cmake/process_options.cmake

-3
Original file line numberDiff line numberDiff line change
@@ -78,9 +78,6 @@ if(ENABLE_NATIVE)
7878
target_compile_options(lightning_compile_options INTERFACE -march=native)
7979
endif()
8080

81-
if(ENABLE_MPI)
82-
set(ENABLE_OPENMP OFF)
83-
endif()
8481

8582
if(PLLGPU_DISABLE_CUDA_SAFETY)
8683
target_compile_options(lightning_compile_options INTERFACE $<$<COMPILE_LANGUAGE:CXX>:-DCUDA_UNSAFE>)

‎pennylane_lightning/core/_version.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,4 +16,4 @@
1616
Version number (major.minor.patch[-label])
1717
"""
1818

19-
__version__ = "0.33.0-dev22"
19+
__version__ = "0.33.0-dev23"

‎pennylane_lightning/core/src/algorithms/CMakeLists.txt

+4
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ project(lightning_algorithms LANGUAGES CXX)
44

55
add_library(lightning_algorithms INTERFACE)
66

7+
if(ENABLE_MPI)
8+
target_compile_options(lightning_compile_options INTERFACE "-D_ENABLE_MPI=1")
9+
endif()
10+
711
target_include_directories(lightning_algorithms INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})
812
target_link_libraries(lightning_algorithms INTERFACE lightning_compile_options
913
lightning_external_libs

‎pennylane_lightning/core/src/algorithms/tests/CMakeLists.txt

+26
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,29 @@ target_link_libraries(algorithms_test_runner PRIVATE algorithms_tests)
4343
catch_discover_tests(algorithms_test_runner)
4444

4545
install(TARGETS algorithms_test_runner DESTINATION bin)
46+
47+
if(ENABLE_MPI)
48+
add_library(algorithms_tests_mpi INTERFACE)
49+
target_link_libraries(algorithms_tests_mpi INTERFACE Catch2::Catch2)
50+
foreach(BACKEND ${PL_BACKEND})
51+
target_link_libraries(algorithms_tests_mpi INTERFACE ${BACKEND}
52+
"${BACKEND}_gates"
53+
"${BACKEND}_measurements"
54+
"${BACKEND}_algorithms"
55+
"${BACKEND}_observables"
56+
)
57+
endforeach()
58+
59+
ProcessTestOptions(algorithms_tests_mpi)
60+
61+
target_sources(algorithms_tests_mpi INTERFACE ./mpi/runner_algorithms_mpi.cpp)
62+
63+
################################################################################
64+
# Define targets
65+
################################################################################
66+
set(TEST_SOURCES ./mpi/Test_AdjointJacobianMPI.cpp)
67+
68+
add_executable(algorithms_test_runner_mpi ${TEST_SOURCES})
69+
target_link_libraries(algorithms_test_runner_mpi PRIVATE algorithms_tests_mpi)
70+
catch_discover_tests(algorithms_test_runner_mpi)
71+
endif()

0 commit comments

Comments
 (0)