diff --git a/.gitignore b/.gitignore index 3f9d398e..ff1ce006 100644 --- a/.gitignore +++ b/.gitignore @@ -11,9 +11,12 @@ *.synctex.gz *.DS_Store *.xwm +*.mw datetime-defaults.sty datetime.sty check-openpmix figs/~$drawings.pptx *.loc *.soc +_minted* +sources/_autogen_/ diff --git a/Makefile b/Makefile index 32ff6660..8c4f896d 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ # Makefile for the PMIx Standard document in LaTex format. # For more information, see the master document, pmix-standard.tex. +LATEX_C=pdflatex -shell-escape -file-line-error + version=4.0 default: pmix-standard.pdf @@ -27,11 +29,8 @@ CHAPTERS= \ App_Python.tex \ Acknowledgements.tex -SOURCES= -# SOURCES=sources/*.c \ -# sources/*.cpp \ -# sources/*.f90 \ -# sources/*.f +SOURCES=sources/*.c \ + sources/*.py \ INTERMEDIATE_FILES=pmix-standard.pdf \ pmix-standard.toc \ @@ -45,27 +44,34 @@ INTERMEDIATE_FILES=pmix-standard.pdf \ pmix-standard.blg \ pmix-standard.synctex.gz \ pmix-standard.xwm \ - *.idx *.ilg *.ind + pmix-standard.mw \ + pmix-standard.loc \ + pmix-standard.soc \ + *.idx *.ilg *.ind \ + _minted-* \ + sources/_autogen_ all: pmix-standard.pdf pmix-standard.pdf: $(CHAPTERS) $(SOURCES) pmix.sty pmix-standard.tex figs/pmix-logo.png - rm -f $(INTERMEDIATE_FILES) + rm -rf $(INTERMEDIATE_FILES) @echo "-------------------------------------------------------------" @echo "If error occurs check pmix-standard.log and pmix-standard.ind" @echo "-------------------------------------------------------------" + @echo "====> Preprocess Examples" + @./bin/process-example.py $(SOURCES) @echo "====> Building 1/4" - pdflatex -interaction=batchmode -file-line-error pmix-standard.tex || \ - pdflatex -interaction=errorstopmode -file-line-error pmix-standard.tex < /dev/null + $(LATEX_C) -interaction=batchmode pmix-standard.tex || \ + $(LATEX_C) -interaction=errorstopmode pmix-standard.tex < /dev/null @echo "====> Building 2/4 (bibtex)" bibtex pmix-standard < /dev/null @echo "====> Building 3/4" - pdflatex -interaction=batchmode -file-line-error pmix-standard.tex || \ - pdflatex -interaction=errorstopmode -file-line-error pmix-standard.tex < /dev/null + $(LATEX_C) -interaction=batchmode pmix-standard.tex || \ + $(LATEX_C) -interaction=errorstopmode pmix-standard.tex < /dev/null @echo "====> Building 4/4" - pdflatex -interaction=batchmode -file-line-error pmix-standard.tex || \ - pdflatex -interaction=errorstopmode -file-line-error pmix-standard.tex < /dev/null - pdflatex -interaction=batchmode -file-line-error pmix-standard.tex + $(LATEX_C) -interaction=batchmode pmix-standard.tex || \ + $(LATEX_C) -interaction=errorstopmode pmix-standard.tex < /dev/null + $(LATEX_C) -interaction=batchmode pmix-standard.tex @./bin/check-doc.sh @echo "====> Success" @cp pmix-standard.pdf pmix-standard-${version}.pdf @@ -93,4 +99,4 @@ check-openpmix: pmix-standard.pdf FORCECHECK @./bin/check-openpmix.py clean: - rm -f $(INTERMEDIATE_FILES) pmix-standard-*.pdf + rm -rf $(INTERMEDIATE_FILES) pmix-standard-*.pdf diff --git a/README.md b/README.md index f0deaa1d..704e007a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ High Performance Computing (HPC) libraries and programming models with portable and well-defined access to commonly available distributed computing system services. -This repository contains the LaTeX source for the PMIx standard document +This repository contains the LaTeX source for the PMIx standard document and discussions regarding the PMIx standard. ## Contributing to the PMIx Standard @@ -15,8 +15,8 @@ and discussions regarding the PMIx standard. We welcome participation in development of the PMIx Standard. To find out more about how to participate, visit our web site at https://pmix.org/ -On the web site, you will find [background documents and -publications](https://pmix.org/publications/), and information +On the web site, you will find [background documents and +publications](https://pmix.org/publications/), and information about [joining our mailing lists, meetings, and working groups](https://pmix.org/contribute/). Additionally, you will find our governance document, @@ -24,3 +24,22 @@ Additionally, you will find our governance document, rules for participation and making changes to the PMIx Standard. The governance document includes instructions for how to raise questions about the standard as well as how to propose changes to the standard. + +## Building the PMIx Standard + +### Building with Docker + +The community maintains a [Docker container image](https://github.com/jjhursey/pmix-standard-dockerfile) that can be used to build the standard using the included script. This is the same container image that is used in CI. + +``` +./bin/build-with-docker.sh +``` + +### Building on Your System + + * If you are building on Ubuntu then take a look at [this Dockerfile](https://github.com/jjhursey/pmix-standard-dockerfile/blob/master/Dockerfile) + * If you are building on Mac OSX start with [MacTeX](https://www.tug.org/mactex/) then add the rest of the dependencies: +``` +sudo pip2 install --upgrade pip setuptools +sudo pip2 install Pygments +``` diff --git a/bin/build-with-docker.sh b/bin/build-with-docker.sh new file mode 100755 index 00000000..913ec89f --- /dev/null +++ b/bin/build-with-docker.sh @@ -0,0 +1,14 @@ +#!/bin/bash -xe + +if [ ! -f "pmix-standard.tex" ] ; then + echo "Error: Must be in the top pmix-standard directory" + exit 1 +fi + +docker pull jjhursey/pmix-standard + +docker run --rm \ + --user $(id -u):$(id -g) \ + -v $PWD:/home/pmixer/doc \ + jjhursey/pmix-standard \ + ./bin/build-std.sh /home/pmixer/doc inplace diff --git a/bin/process-example.py b/bin/process-example.py new file mode 100755 index 00000000..4d527051 --- /dev/null +++ b/bin/process-example.py @@ -0,0 +1,188 @@ +#!/usr/bin/python -u + +# +# Recognized tags +# : Start a block +# : End a block +# Rules: +# - The tag strings, regardless of ID, are removed from the final output +# - If multiple blocks with the same ID exist then they are concatinated together +# - ID can contain alphabetic and numberic characters and the following symbols: . _ +# - Quote marks are stripped out +# +import sys +import os +import re +import argparse +import subprocess +import shutil + +EG_BEGIN_STR="EG BEGIN ID=" +EG_END_STR="EG END ID=" +EG_ID_PATTERN="[A-Za-z0-9_\.\"]" + + +class Example: + """An example from the source code""" + eid = None + active = False + filename = None + code_block = "" + + def __init__(self): + self.eid = "" + self.active = False + self.filename = None + self.code_block = "" + + def __str__(self): + return "in_fname=[%s] id=[%s]" % (self.filename, self.eid) + + def get_out_fname(self): + f_id = self.eid.replace('"', '') + return os.path.basename(self.filename) + "_" + f_id + + def append_line(self, line): + self.code_block = self.code_block + line + if line.endswith("\n") is False: + self.code_block = self.code_block + "\n" + + def get_code_block(self): + final_block = "" + min_lead_spaces = 10000 + lines = 0 + total_lines = 0 + skip_last_lines = 0 + + # First pass to find min spacing + for line in self.code_block.splitlines(True): + if re.search(r"\s*"+EG_BEGIN_STR, line) is not None: + continue + if re.search(r"\s*"+EG_END_STR, line) is not None: + continue + + total_lines = total_lines + 1 + # Skip empty lines + if len(line) <= 1: + skip_last_lines = skip_last_lines + 1 + continue + else: + skip_last_lines = 0 + + m = re.match(r"^([ \t]+)", line) + if m is not None: + c_len = len(m.group(1)) + if c_len > 1: + min_lead_spaces = min(c_len, min_lead_spaces) + else: + # Indicates that there is a line that does not have leading spaces, but is not empty + min_lead_spaces = 0 + + # Next pass to build the string + lines = 0 + for line in self.code_block.splitlines(True): + if re.search(r"\s*"+EG_BEGIN_STR, line) is not None: + continue + if re.search(r"\s*"+EG_END_STR, line) is not None: + continue + + # Clear off trailing empty lines + if total_lines - skip_last_lines == lines: + break + lines = lines + 1 + + m = re.match(r"^([ \t]+)", line) + if m is not None: + line = line[min_lead_spaces:] + final_block = final_block + line + + return final_block + + +def process_file(filename): + """Process all of the key/value splitting in the file""" + all_examples = {} + eg_id = None + + with open(filename, 'r') as fd: + for line in fd: + line = line.rstrip() + m = re.search(r"\s*"+EG_BEGIN_STR+"("+EG_ID_PATTERN+"*)", line) + if m is not None: + eg_id = m.group(1) + # Find this object and update it + found = False + for example_id in all_examples: + if example_id == eg_id: + example = all_examples[example_id] + example.active = True + example.append_line(line) + found = True + # Insert it if not found + if found is False: + x = Example() + x.eid = eg_id + x.active = True + x.append_line(line) + x.filename = filename + all_examples[eg_id] = x + continue + + m = re.search(r"\s*"+EG_END_STR+"("+EG_ID_PATTERN+"*)", line) + if m is not None: + eg_id = m.group(1) + # Find this object and update it + for example_id in all_examples: + if example_id == eg_id: + example = all_examples[example_id] + example.active = False + example.append_line("") # Add an empty line + continue + + for example_id in all_examples: + example = all_examples[example_id] + if example.active is True: + example.append_line(line) + + return all_examples + + +if __name__ == "__main__": + # + # Command line parsing + # + parser = argparse.ArgumentParser(description="PMIx Standard Example Preprocessor") + parser.add_argument("-v", "--verbose", action="store_true", help="Verbose output") + parser.add_argument("files", nargs='+', help="List of files to process") + parser.parse_args() + args = parser.parse_args() + + # + # Create a temporary directory to store the snippets + # + gen_dir = "sources/_autogen_" + if os.access(gen_dir, os.W_OK) is False: + os.makedirs(gen_dir) + + # + # Iterate through all examples and split out the snippets + # + for f in args.files: + if os.path.exists(f) is False: + print("ERROR: File does not exist: %s" % (f)) + sys.exit(1) + + print("Processing File: %s" % (f)) + example_blocks = process_file(f) + for k, example in example_blocks.items(): + out_fname = gen_dir + "/" + example.get_out_fname() + print("\tExample: %s -- Stored in %s" % (example, out_fname)) + if args.verbose: + print("CODE BLOCK") + print("-" * 50) + print(example.get_code_block()), + print("-" * 50) + with open(out_fname, 'w') as fd: + fd.write("%s" % (example.get_code_block())) + + sys.exit(0) diff --git a/pmix.sty b/pmix.sty index d89b5bef..34110aac 100644 --- a/pmix.sty +++ b/pmix.sty @@ -760,3 +760,109 @@ \newcounter{pycounter} \newcommand{\pylabel}[1]{\refstepcounter{pycounter} \label{appB:#1}} \newcommand{\refpy}[1]{\hyperref[appB:#1]{\code{#1}}} + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Code examples +% Install: (if using Python 3 then use pip3) +% sudo pip2 install --upgrade pip setuptools +% sudo pip2 install Pygments +% Ubuntu: +% sudo apt update +% sudo apt install python-pip +% sudo pip3 install Pygments +% +% package to work around "No room for a new \write" message +\usepackage{morewrites} +\morewritessetup{allocate=10} +% package for code highlighting +\usepackage[chapter]{minted} +\usemintedstyle{default} +\usepackage{xcolor} + +% Background: Light Gray +\definecolor{pmixbg}{rgb}{0.95,0.95,0.95} +% Highlight: Yellow +\definecolor{pmixhl}{rgb}{0.95,0.95,0} + +%----------------------- +% C Code +% +% block of code +% \begin{pmixCodeC} +% return 0; +% \end{pmixCodeC} +% +% Inline C code +% The code \pmixCodeInlineC{printf("Hello World");} prints ``Hello World''. +% +% Import C code from a file: +% - Whole file: \pmixCodeImportC{sources/hello.c} +% - Selection of a file: \pmixCodeImportC[firstline=73, lastline=84]{sources/hello.c} +% - Highlight selection of a file: \pmixCodeImportC[firstline=73, lastline=84, highlightlines={2-4,6}]{sources/hello.c} +% +\newminted[pmixCodeC]{c}{fontsize=\small, + bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + numbersep=3pt, + firstnumber=1} +\newmintinline[pmixCodeInlineC]{c}{bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + firstnumber=1} +\newmintedfile[pmixCodeImportC]{c}{fontsize=\small, + bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + numbersep=3pt, + firstnumber=1} + +%----------------------- +% Python Code +% +% block of code +% \begin{pmixCodePy} +% print("Hello") +% \end{pmixCodePy} +% +% Inline C code +% The code \pmixCodeInlinePy{print("Hello World")} prints ``Hello World''. +% +% Import C code from a file: +% - Whole file: \pmixCodeImportPy{sources/hello.py} +% - Selection of a file: \pmixCodeImportPy[firstline=73, lastline=84]{sources/hello.py} +% - Highlight selection of a file: \pmixCodeImportPy[firstline=73, lastline=84, highlightlines={2-4,6}]{sources/hello.py} +% +\newminted[pmixCodepPy]{python}{fontsize=\small, + bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + numbersep=3pt, + firstnumber=1} +\newmintinline[pmixCodeInlinePy]{python}{bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + firstnumber=1} +\newmintedfile[pmixCodeImportPy]{python}{fontsize=\small, + bgcolor=pmixbg, + highlightcolor=pmixhl, + breaklines, + autogobble, + linenos, + numbersep=3pt, + firstnumber=1} + + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/sources/README.md b/sources/README.md new file mode 100644 index 00000000..c0223dd7 --- /dev/null +++ b/sources/README.md @@ -0,0 +1,119 @@ +# Source Code Examples + +## General Rules + + * All examples should be buildable + * All examples should contain a comment in the header with: + - How to run the example + - Example output from a run + +## Preprocessor Syntax + +The preprocessor will automatically extract code snippets from the source files. + +The code snippets will be in the `sources/_autogen_` directory which is deleted and recreated on every build. +The code snippets will be named `sources/_autogen_/FILENAME_TAG` where `FILENAME` is the source file and `TAG` is the tag used inside the code snippet to identify the snippet. +Code snippets can span multiple sections which are concatinated, in order of appearance, into the output file. +Code snippets cannot span multiple files. + + * Start a snippet tagged `myexample` : `` + * End a snippet tagged `myexample` : `` + * Tags may only contain alphabetic and numberic characters and the following symbols: `.` `_` + - Quote marks are stripped out of the string + * Lines containing the snippet syntax as stripped out of the generated examples. + +### Example + +Given the following example program called `hello-alt.c` + +``` +#include + +int main(int argc, char **argv) +{ + // + // + pmix_status_t rc = PMIX_SUCCESS; + pmix_proc_t myproc; + // + pmix_value_t *val; + uint16_t localrank; + // + + // + rc = PMIx_Init(&myproc, NULL, 0); + if (PMIX_SUCCESS != rc) { + return 1; + } + // + + // + /* Get our rank local to this node */ + rc = PMIx_Get(&myproc, PMIX_LOCAL_RANK, NULL, 0, &val); + if (PMIX_SUCCESS != rc) { + return 2; + } + localrank = val->data.uint16; + PMIX_VALUE_RELEASE(val); + // + + rc = PMIx_Finalize(NULL, 0); + return rc; +} +``` + +Produces two snippet files + +``` +Processing File: sources/hello-alt.c + Example: in_fname=[sources/hello-alt.c] id=["pmix_get"] -- Stored in sources/_autogen_/hello-alt.c_pmix_get + Example: in_fname=[sources/hello-alt.c] id=["pmix_init"] -- Stored in sources/_autogen_/hello-alt.c_pmix_init +``` + +``` +shell$ cat sources/_autogen_/hello-alt.c_pmix_init +pmix_status_t rc = PMIX_SUCCESS; +pmix_proc_t myproc; + +rc = PMIx_Init(&myproc, NULL, 0); +if (PMIX_SUCCESS != rc) { + return 1; +} +``` + +``` +shell$ cat sources/_autogen_/hello-alt.c_pmix_get +pmix_status_t rc = PMIX_SUCCESS; +pmix_proc_t myproc; +pmix_value_t *val; +uint16_t localrank; + +/* Get our rank local to this node */ +rc = PMIx_Get(&myproc, PMIX_LOCAL_RANK, NULL, 0, &val); +if (PMIX_SUCCESS != rc) { + return 2; +} +localrank = val->data.uint16; +PMIX_VALUE_RELEASE(val); +``` + +## Including Code Snippets in Latex + +Import C code with and without highlighting (line numbers from generated source) +``` +Example without highlighting +\pmixCodeImportC{sources/_autogen_/hello-alt.c_pmix_init}% + +Example with highlighting: +\pmixCodeImportC[highlightlines={2-3,7,11-13}]{sources/_autogen_/hello-alt.c_pmix_get}% + +\pmixCodeInlineC{printf("Hello World");} prints ``Hello World''. +``` + +Import Python code + +``` +The inline code \pmixCodeInlinePy{print("Hello World")} prints ``Hello World''. + +\pmixCodeImportPy{sources/_autogen_/hello.py_pmix_py}% +``` \ No newline at end of file diff --git a/sources/hello.c b/sources/hello.c new file mode 100644 index 00000000..3c8cede9 --- /dev/null +++ b/sources/hello.c @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * Copyright (c) 2020 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include + +int main(int argc, char **argv) +{ + pid_t pid; + char hostname[1024]; + pmix_status_t rc = PMIX_SUCCESS; + pmix_proc_t myproc; + pmix_value_t *val; + uint16_t localrank; + + pid = getpid(); + gethostname(hostname, 1024); + + /* Initialize PMIx - client role */ + rc = PMIx_Init(&myproc, NULL, 0); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + return 1; + } + + /* Get our rank local to this node */ + rc = PMIx_Get(&myproc, PMIX_LOCAL_RANK, NULL, 0, &val); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, + "Client ns %s rank %d: PMIx_Get local rank failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + localrank = val->data.uint16; + PMIX_VALUE_RELEASE(val); + + printf("Client ns %s rank %d pid %lu: Running on host %s localrank %d\n", + myproc.nspace, myproc.rank, (unsigned long)pid, hostname , + (int)localrank); + + /* Fence to hold all processes in this namespace */ + pmix_proc_t wildproc; + PMIX_PROC_CONSTRUCT(&wildproc); + (void)strncpy(wildproc.nspace, myproc.nspace, PMIX_MAX_NSLEN); + wildproc.rank = PMIX_RANK_WILDCARD; + + rc = PMIx_Fence(&wildproc, 1, NULL, 0); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, + "Client ns %s rank %d: PMIx_Fence failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + goto done; + } + + done: + /* Finalize the PMIx library */ + printf("Client ns %s rank %d: Finalizing\n", myproc.nspace, myproc.rank); + rc = PMIx_Finalize(NULL, 0); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, "Client ns %s rank %d:PMIx_Finalize failed: %s\n", + myproc.nspace, myproc.rank, PMIx_Error_string(rc)); + } else { + printf("Client ns %s rank %d:PMIx_Finalize successfully completed\n", + myproc.nspace, myproc.rank); + } + + return rc; +} diff --git a/sources/hello.py b/sources/hello.py new file mode 100755 index 00000000..1b5e001e --- /dev/null +++ b/sources/hello.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +from pmix import * + +def main(): + # + client_hdl = PMIxClient() + # + print("Testing PMIx ", client_hdl.get_version()) + + print("Init") + # + info = [] + (rc,myproc) = client_hdl.init(info) + if 0 != rc: + exit(1) + # + + print("Get") + info = [] + rc, get_val = client_hdl.get(myproc, "PMIX_LOCAL_RANK", info) + print("Get result: ", rc) + print("Get value returned: ", get_val) + + print("Finalize") + info = [] + client_hdl.finalize(info) + +if __name__ == '__main__': + main()