1991 better compilation support

doc/developer_guide/psy_data.rst

@@ -39,6 +39,9 @@
 
 .. testsetup::
 
+    import os
+    from psyclone.parse import ModuleManager
+
     # Define SOURCE_FILE to point to an existing gocean 1.0 file.
     SOURCE_FILE = ("../../src/psyclone/tests/test_files/"
         "gocean1p0/test11_different_iterates_over_one_invoke.f90")
@@ -1214,3 +1217,71 @@ compared with the expected values in ``field1_post``.
 
 .. note:: For now the created driver still depends on the infrastructure
     library and any other modules used. Issue #1991 improves this.
+
+Module Manager
+++++++++++++++
+The LFRic driver creation utilises a ``ModuleManager`` to find
+and inline all modules required by the driver.
+
+
+.. autoclass:: psyclone.parse.ModuleManager
+    :members:
+
+Any PSyclone command line option ``-d`` (see :ref:`psyclone_command`)
+will be added to the ``ModuleManager`` as recursive search paths. The
+``ModuleManager`` is a singleton and it can be queried for information about
+any module. It internally uses caching to avoid repeatedly searching
+directories, and it will only access search paths as required. For example,
+if the first search path will be sufficient to find all modules during the
+lifetime of the module manager, no other search path will ever be accessed.
+The caching also implies that the ModuleManager will not detect if new files
+should be created during its lifetime.
+
+The ``ModuleManager`` also provides a static function that will sort
+a list of module dependencies, so that compiling the modules in this order
+(or adding them in this order to a file) will allow compilation, i.e. any
+module will only depend on previously defined modules.
+
+
+The ``ModuleManager`` will return a ``ModuleInfo`` object to make information
+about a module available:
+
+.. autoclass:: psyclone.parse.ModuleInfo
+    :members:
+
+Similar to the ``ModuleManager``, a ``ModuleInfo`` object will heavily rely on
+caching to avoid repeatedly reading a source file or parsing it. The side
+effect is that changes to a source file during the lifetime of the
+``ModuleManager`` will not be reflected in its information.
+
+At this stage, the ``ModuleInfo`` can be used to get the original source
+code of a module as string and to query a module about modules and symbols
+it depends on. It uses the fparser parse tree to detect this information (which
+means it can handle files that are not supported by PSyIR, e.g. files with
+preprocessor directives).
+
+An example usage of the ``ModuleManager`` and ``ModuleInfo`` objects,
+which prints the filenames of all modules used in ``tl_testkern_mod``:
+
+.. testcode ::
+
+    mod_manager = ModuleManager.get()
+    # Add the path to the PSyclone LFRic example codes:
+    mod_manager.add_search_path("../../src/psyclone/tests/test_files/"
+                                "dynamo0p3")
+
+    testkern_info = mod_manager.get_module_info("tl_testkern_mod")
+
+    used_mods = testkern_info.get_used_modules()
+    # Sort the modules so we get a reproducible output ordering
+    used_mods_list = sorted(list(used_mods))
+    for module_name in used_mods_list:
+        mod_info = mod_manager.get_module_info(module_name)
+        print("Module:", module_name, os.path.basename(mod_info.filename))
+
+.. testoutput::
+
+    Module: argument_mod argument_mod.f90
+    Module: constants_mod constants_mod.f90
+    Module: fs_continuity_mod fs_continuity_mod.f90
+    Module: kernel_mod kernel_mod.f90

doc/reference_guide/source/.gitignore

@@ -0,0 +1 @@
+doxygen

doc/user_guide/examples.rst

@@ -73,7 +73,7 @@ target checks the various Jupyter notebooks using ``nbconvert``.
           ``examples`` directory but still wish to use ``make`` then you
           will also have to set the ``PSYCLONE_CONFIG`` environment variable
           to the full path to the PSyclone configuration file, e.g.
-          ``$ PSYCLONE_CONFIG=/some/path/psyclone.cfg make``.
+          ``PSYCLONE_CONFIG=/some/path/psyclone.cfg make``.
 
 .. _examples-compilation:
 
@@ -574,24 +574,35 @@ Example 17.3: Kernel Data Extraction
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The example in the subdirectory ``full_example_extract`` shows the
-use of :ref:`kernel extraction <psyke>`. It requires the
-installation of a NetCDF development environment (see
-`here
+use of :ref:`kernel extraction <psyke>`. The code can be compiled with
+``make compile``, and the binary executed with either ``make run`` or
+``./extract.standalone``. By default, it will be using
+a stand-alone extraction library (see :ref:`extraction_libraries`).
+If you want to use the NetCDF version, set the environment variable
+``TYPE`` to be ``netcdf``:
+
+.. code-block:: bash
+
+    TYPE=netcdf make compile
+
+This requires the installation of a NetCDF development environment
+(see `here
 <https://github.com/stfc/PSyclone/tree/master/tutorial/practicals#user-content-netcdf-library-lfric-examples>`_
-for installing NetCDF).
-The code can be compiled with ``make compile``, and
-the binary executed with either ``make run`` or ``./extract``
-Running the compiled binary will create one NetCDF file ``main-update.nc``
-containing the input and output parameters for the ``testkern_w0``
-kernel call. For example:
+for installing NetCDF). The binary will be called ``extract.netcdf``,
+and the output files will have the ``.nc`` extension.
+
+Running the compiled binary will create two Fortran binary files or
+two NetCDF files if the NetCDF library was used. They contain
+the input and output parameters for the two invokes in this example:
 
 .. code-block:: bash
 
     cd full_example_extraction
-    make compile
-    ./extract
+    TYPE=netcdf make compile
+    ./extract.netcdf
     ncdump ./main-update.nc | less
 
+
 Example 18: Special Accesses of Continuous Fields - Incrementing After Reading and Writing Before (Potentially) Reading
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

doc/user_guide/psyke.rst

@@ -479,7 +479,7 @@ be set to true:
                   {"create_driver": True,
                    "region_name": ("main", "init")})
 
-This will create a Fortran file called ``driver-main-init.f90``, which
+This will create a Fortran file called ``driver-main-init.F90``, which
 can then be compiled and executed. This stand-alone program will read
 the output file created during an execution of the actual program, call
 the kernel with all required input parameter, and compare the output
@@ -503,10 +503,31 @@ optimisation of a stand-alone kernel.
     run stand-alone. As a work-around, these values can be added manually
     to the driver program. Issue #1990 tracks improvement of this situation.
 
-When linking the driver program, it needs to be provided with all
-dependencies required by the driver and the kernel used. If the kernel calls
-many other functions, this can result in a long parameter list for the
-linker. Issue #1991 aims at simplifying this.
+The LFRic kernel driver will inline all required external modules into the
+driver. It uses a ``ModuleManager`` to find the required modules, based on the
+assumption that a file ``my_special_mod.f90`` will define exactly one module
+called ``my_special_mod`` (the ``_mod`` is required to be part of the
+filename). The driver creator will sort the modules in the appropriate order
+and add the source code directly into the driver. As a result, the driver
+program is truly stand-alone and does not need any external dependency (the
+only exception being NetCDF if the NetCDF-based extraction library is used).
+The ``ModuleManager`` uses all kernel search paths specified on the
+command line (see ``-d`` option in :ref:`psyclone_command`), and it will
+recursively search for all files under each path specified on the command
+line.
+
+Therefore, compilation for a created driver, e.g. the one created in
+``examples/lfric/eg17/full_example_extract``, is simple::
+
+   $ gfortran -g -O0 driver-main-update.F90 -o driver-main-update
+   $ ./driver-main-update
+   cell correct
+   field1 correct
+
+Note that the Makefile in the example will actually provide additional include
+paths (infrastructure files and extraction library) for the compiler, but
+these flags are actually only required for compiling the example program, not
+for the driver.
 
 
 Extraction for NEMO

examples/lfric/eg17/full_example_extract/Makefile

@@ -51,6 +51,9 @@ ifeq ($(TYPE), netcdf)
 else
 	GENERATED_FILES += main-update.binary main-init.binary
 endif
+GENERATED_FILES += driver-main-init driver-main-init.F90       \
+				   driver-main-update driver-main-update.F90   \
+				   *.o *.mod $(EXEC) main_alg.f90 main_psy.f90
 
 F90 ?= gfortran
 F90FLAGS ?= -Wall -g -ffree-line-length-none
@@ -60,12 +63,12 @@ OBJ = main_psy.o main_alg.o testkern_w0_kernel_mod.o
 ifeq ($(TYPE), netcdf)
 	EXTRACT_DIR ?= $(PSYROOT)/lib/extract/netcdf/lfric
 	LDFLAGS += $$(nf-config --flibs)
+	F90FLAGS += $$(nf-config --fflags)
 else
 	EXTRACT_DIR ?= $(PSYROOT)/lib/extract/standalone/lfric
 endif
 
 EXEC = extract.$(TYPE)
-GENERATED_FILES = *.o *.mod  $(EXEC) main_alg.f90 main_psy.f90
 EXTRACT_NAME ?= _extract
 EXTRACT_LIB = $(EXTRACT_DIR)/lib$(EXTRACT_NAME).a
 READ_KERNEL_DATA_OBJ = $(EXTRACT_DIR)/../read_kernel_data_mod.o
@@ -83,7 +86,7 @@ transform: main_psy.f90 main_alg.f90
 compile: transform $(EXEC)
 
 run: compile
-	./extract
+	./$(EXEC)
 
 $(EXEC): $(LFRIC_LIB) $(EXTRACT_LIB) $(OBJ)
 	$(F90) $(F90FLAGS) $(LFRIC_INCLUDE_FLAGS) $(OBJ) -o $(EXEC) -L$(EXTRACT_DIR) -l$(EXTRACT_NAME) \
@@ -103,18 +106,12 @@ testkern_w0_kernel_mod.o: $(LFRIC_LIB)
 driver-main-update:	LFRIC_INCLUDE_FLAGS += -I $(EXTRACT_DIR)/..
 driver-main-update:	driver-main-update.o $(READ_KERNEL_DATA_OBJ)
 	$(F90) $(F90FLAGS) $(LFRIC_INCLUDE_FLAGS)  driver-main-update.o \
-		testkern_w0_kernel_mod.o                   \
-		$(READ_KERNEL_DATA_OBJ)                    \
-		-L$(LFRIC_PATH) -l$(LFRIC_NAME) $(LDFLAGS) \
-		-o driver-main-update
+		   $(LDFLAGS) -o driver-main-update
 
 driver-main-init:	LFRIC_INCLUDE_FLAGS += -I $(EXTRACT_DIR)/..
 driver-main-init:	driver-main-init.o $(READ_KERNEL_DATA_OBJ)
 	$(F90) $(F90FLAGS) $(LFRIC_INCLUDE_FLAGS)  driver-main-init.o \
-		testkern_w0_kernel_mod.o                   \
-		$(READ_KERNEL_DATA_OBJ)                    \
-		-L$(LFRIC_PATH) -l$(LFRIC_NAME) $(LDFLAGS) \
-		-o driver-main-init
+		   $(LDFLAGS) -o driver-main-init
 
 driver-main-update.o driver-main-init.o:  $(READ_KERNEL_DATA_OBJ)
 
@@ -133,7 +130,9 @@ $(READ_KERNEL_DATA_OBJ):
 	$(MAKE) -C $(EXTRACT_DIR)/..
 
 %_psy.f90:	%.x90
-	${PSYCLONE} -s ./extract_transform.py \
+	${PSYCLONE} -s ./extract_transform.py       \
+	-d . -d $(EXTRACT_DIR) -d $(EXTRACT_DIR)/.. \
+	-d $(LFRIC_PATH)                            \
 	-nodm -opsy $*_psy.f90 -oalg $*_alg.f90 $<
 
 allclean: clean

lib/extract/netcdf/.gitignore

@@ -1,3 +1,2 @@
 extract_netcdf_base.f90
-read_kernel_data_mod.f90