Logging for SmartRedis

.github/workflows/run_post_merge_tests.yml

@@ -140,5 +140,7 @@ jobs:
         if: contains(matrix.os, 'ubuntu') && !contains(matrix.compiler, 'intel')
         run: |
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/install/lib &&
+          export SR_LOG_FILE=smartredis_examples_log.txt &&
+          export SR_LOG_LEVEL=INFO &&
           make test-examples
 

.github/workflows/run_tests.yml

@@ -107,6 +107,8 @@ jobs:
             bash ../build-scripts/build-catch.sh &&
             cd ../ &&
             export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/install/lib &&
+            export SR_LOG_FILE=smartredis_cicd_tests_log.txt &&
+            export SR_LOG_LEVEL=INFO &&
             make test-verbose
 
       - name: Run Python coverage tests
@@ -163,5 +165,7 @@ jobs:
       - name: Run testing with redis cluster
         run: |
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/install/lib &&
+          export SR_LOG_FILE=smartredis_cicd_tests_log.txt &&
+          export SR_LOG_LEVEL=INFO &&
           make test-verbose
 

.github/workflows/run_tests_uds.yml

@@ -145,5 +145,7 @@ jobs:
       - name: Run testing with UDS redis
         run: |
           export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$PWD/install/lib &&
+          export SR_LOG_FILE=smartredis_uds_tests_log.txt &&
+          export SR_LOG_LEVEL=INFO &&
           make test-verbose
 

CMakeLists.txt

@@ -59,6 +59,7 @@ set(CLIENT_SRC
     src/c/c_client.cpp
     src/c/c_dataset.cpp
     src/c/c_error.cpp
+    src/c/c_logcontext.cpp
     src/cpp/address.cpp
     src/cpp/client.cpp
     src/cpp/dataset.cpp
@@ -87,6 +88,9 @@ set(CLIENT_SRC
     src/cpp/stringfield.cpp
     src/cpp/pipelinereply.cpp
     src/cpp/threadpool.cpp
+    src/cpp/utility.cpp
+    src/cpp/logger.cpp
+    src/cpp/srobject.cpp
 )
 
 include_directories(SYSTEM
@@ -96,10 +100,12 @@ include_directories(SYSTEM
 
 if (BUILD_FORTRAN)
     set(FORTRAN_SRC
-        src/fortran/fortran_c_interop.F90
-        src/fortran/dataset.F90
         src/fortran/client.F90
+        src/fortran/dataset.F90
         src/fortran/errors.F90
+        src/fortran/fortran_c_interop.F90
+        src/fortran/logcontext.F90
+        src/fortran/logger.F90
     )
     include_directories(src/fortran)
     # Note the following has to be before ANY add_library command)
@@ -137,6 +143,8 @@ if(BUILD_PYTHON)
     add_library(smartredis_static STATIC ${CLIENT_SRC})
 
 	pybind11_add_module(smartredisPy
+                        src/python/src/pysrobject.cpp
+                        src/python/src/pylogcontext.cpp
 	                    src/python/src/pyclient.cpp
                         src/python/src/pydataset.cpp
                         ${CLIENT_SRC}

conftest.py

@@ -65,6 +65,10 @@ def mock_data():
 def mock_model():
     return MockTestModel
 
+@pytest.fixture
+def context(request):
+    return request.node.name
+
 class MockTestData:
 
     @staticmethod

doc/changelog.rst

@@ -14,6 +14,7 @@ Description
 
 - Improved error reporting capabilities for Fortran clients
 - Python error messages from SmartRedis contain more information
+- Added logging functionality to the SmartRedis library
 - A bug related to thread pool initialization was fixed.
 - This version adds new functionality in the form of support for Unix Domain Sockets.
 - Fortran client can now be optionally built with the rest of the library
@@ -23,6 +24,7 @@ Detailed Notes
 
 - Fortran clients can now access error text and source location (PR284_)
 - Add exception location information from CPP code to Python exceptions (PR283_)
+- Added client activity and manual logging for developer use (PR281_)
 - Fix thread pool error (PR280_)
 - Update library linking instructions and update Fortran tester build process (PR277_)
 - Added `add_metadata_for_xarray` and `transform_to_xarray` methods in `DatasetConverter` class for initial support with Xarray (PR262_)
@@ -32,6 +34,7 @@ Detailed Notes
 
 .. _PR280: https://github.com/CrayLabs/SmartRedis/pull/284
 .. _PR280: https://github.com/CrayLabs/SmartRedis/pull/283
+.. _PR281: https://github.com/CrayLabs/SmartRedis/pull/281
 .. _PR280: https://github.com/CrayLabs/SmartRedis/pull/280
 .. _PR277: https://github.com/CrayLabs/SmartRedis/pull/277
 .. _PR262: https://github.com/CrayLabs/SmartRedis/pull/262

doc/runtime.rst

@@ -42,6 +42,36 @@ location.  However, the Python ``Client`` constructor also allows
 for the database location to be set as an input parameter. In
 this case, it sets SSDB from the input parameter.
 
+Logging Environment Variables
+=============================
+
+SmartRedis will log events to a file on behalf of a client. There
+are two main environment variables that affect logging, ``SR_LOG_FILE``
+and ``SR_LOG_LEVEL``.
+
+``SR_LOG_FILE`` is the specifier for the location of the file that
+receives logging information. Each entry in the file will be prefixed
+with a timestamp and the identifier of the client that invoked the logging
+message. The path may be relative or absolute, though a relative path risks
+creation of multiple log files if the executables that instantiate SmartRedis
+clients are run from different directories. If this variable is not set,
+logging information will be sent to standard console output (STDOUT in C and
+C++; output_unit in Fortran).
+
+``SR_LOG_LEVEL`` relates to the verbosity of information that wil be logged.
+It may be one of three levels: ``QUIET`` disables logging altogether.
+``INFO`` provides informational logging, such as exception events that
+transpire within the SmartRedis library and creation or destruction of a
+client object.  ``DEBUG`` provides more verbose logging, including information
+on the activities of the SmartRedis thread pool and API function entry and exit.
+Debug level logging will also log the absence of an expected environment variable,
+though this can happen only if the variables to set up logging are in place. If
+this parameter is not set, a default logging level of ``INFO`` will be adopted.
+
+The runtime impact of log levels NONE or INFO should be minimal on
+client performance; however, seting the log level to DEBUG may cause some
+degradation.
+
 Ensemble Environment Variables
 ==============================
 

examples/parallel/cpp/smartredis_mnist.cpp

@@ -100,10 +100,12 @@ int main(int argc, char* argv[]) {
     // Retrieve the MPI rank
     int rank;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    std::string logger_name("Client ");
+    logger_name += std::to_string(rank);
 
     // Initialize a Client object
     bool cluster_mode = true; // Set to false if not using a clustered database
-    SmartRedis::Client client(cluster_mode);
+    SmartRedis::Client client(cluster_mode, logger_name);
 
     // Set the model and script that will be used by all ranks
     // from MPI rank 0.

examples/parallel/cpp/smartredis_put_get_3D.cpp

@@ -50,13 +50,17 @@ int main(int argc, char* argv[]) {
     for(size_t i=0; i<n_values; i++)
         input_tensor[i] = 2.0*rand()/RAND_MAX - 1.0;
 
+    // Get our rank
+    int rank = 0;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    std::string logger_name("Client ");
+    logger_name += std::to_string(rank);
+
     // Initialize a SmartRedis client
     bool cluster_mode = true; // Set to false if not using a clustered database
-    SmartRedis::Client client(cluster_mode);
+    SmartRedis::Client client(cluster_mode, logger_name);
 
     // Put the tensor in the database
-    int rank = 0;
-    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     std::string key = "3d_tensor_" + std::to_string(rank);
     client.put_tensor(key, input_tensor.data(), dims,
                       SRTensorTypeDouble, SRMemLayoutContiguous);

examples/parallel/fortran/smartredis_dataset.F90

@@ -85,7 +85,7 @@ program main
   if (result .ne. SRNoError) error stop 'dataset%get_meta_scalars failed'
 
   ! Initialize a client
-  result = client%initialize(.true.) ! Change .false. to .true. if not using a clustered database
+  result = client%initialize(.true., "smartredis_dataset") ! Change .false. to .true. if not using a clustered database
   if (result .ne. SRNoError) error stop 'client%initialize failed'
 
   ! Send the dataset to the database via the client

examples/parallel/fortran/smartredis_mnist.F90

@@ -51,7 +51,7 @@ program mnist_example
   write(key_suffix, "(A,I1.1)") "_",pe_id
 
   ! Initialize a client
-  result = client%initialize(.true.) ! Change .false. to .true. if not using a clustered database
+  result = client%initialize(.true., "smartredis_mnist") ! Change .false. to .true. if not using a clustered database
   if (result .ne. SRNoError) error stop 'client%initialize failed'
 
   ! Set up model and script for the computation

examples/parallel/fortran/smartredis_put_get_3D.F90

@@ -57,7 +57,7 @@ program main
   call random_number(recv_array_real_64)
 
   ! Initialize a client
-  result = client%initialize(.true.) ! Change .false. to .true. if not using a clustered database
+  result = client%initialize(.true., "smartredis_put_get_3D") ! Change .false. to .true. if not using a clustered database
   if (result .ne. SRNoError) error stop 'client%initialize failed'
 
   ! Add a tensor to the database and verify that we can retrieve it

examples/serial/c/example_put_get_3D.c

@@ -38,6 +38,8 @@ int main(int argc, char* argv[]) {
   values.
   */
 
+  const char* logger_name = "put_get_3d";
+  size_t cid_len = strlen(logger_name);
   size_t n_dims = 3;
   size_t* dims = malloc(n_dims*sizeof(size_t));
   dims[0] = 10;
@@ -46,7 +48,7 @@ int main(int argc, char* argv[]) {
 
   void* client = NULL;
   bool cluster_mode = true; // Set to false if not using a clustered database
-  if (SRNoError != SmartRedisCClient(cluster_mode, &client)) {
+  if (SRNoError != SmartRedisCClient(cluster_mode, logger_name, cid_len, &client)) {
     printf("Client initialization failed!\n");
     exit(-1);
   }

examples/serial/c/example_put_unpack_1D.c

@@ -34,6 +34,8 @@
 
 int main(int argc, char* argv[]) {
 
+  const char* logger_name = "put_unpack_1d";
+  size_t cid_len = strlen(logger_name);
   size_t* dims = malloc(sizeof(size_t));
   dims[0] = 10;
   size_t n_dims = 1;
@@ -46,7 +48,7 @@ int main(int argc, char* argv[]) {
 
   void* client = NULL;
   bool cluster_mode = true; // Set to false if not using a clustered database
-  if (SRNoError != SmartRedisCClient(cluster_mode, &client)) {
+  if (SRNoError != SmartRedisCClient(cluster_mode, logger_name, cid_len, &client)) {
     printf("Client initialization failed!\n");
     exit(-1);
   }

examples/serial/cpp/smartredis_dataset.cpp

@@ -56,7 +56,7 @@ int main(int argc, char* argv[]) {
 
     // Initialize a SmartRedis client
     bool cluster_mode = true; // Set to false if not using a clustered database
-    SmartRedis::Client client(cluster_mode);
+    SmartRedis::Client client(cluster_mode, __FILE__);
 
     // Create a DataSet
     SmartRedis::DataSet dataset("example_dataset");

examples/serial/cpp/smartredis_mnist.cpp

@@ -78,7 +78,7 @@ int main(int argc, char* argv[]) {
     // In general, Client objects should be reused, but for this
     // small example, Client objects are not reused.
     bool cluster_mode = true; // Set to false if not using a clustered database
-    SmartRedis::Client client(cluster_mode);
+    SmartRedis::Client client(cluster_mode, __FILE__);
 
     // Build model key, file name, and then set model
     // from file using client API

examples/serial/cpp/smartredis_model.cpp

@@ -48,7 +48,7 @@ int main(int argc, char* argv[]) {
 
     // Initialize a SmartRedis client to connect to the Redis database
     bool cluster_mode = true; // Set to false if not using a clustered database
-    SmartRedis::Client client(cluster_mode);
+    SmartRedis::Client client(cluster_mode, __FILE__);
 
     // Use the client to set a model in the database from a file
     std::string model_key = "mnist_model";

examples/serial/cpp/smartredis_put_get_3D.cpp

@@ -48,7 +48,7 @@ int main(int argc, char* argv[]) {
 
     // Initialize a SmartRedis client
     bool cluster_mode = true; // Set to false if not using a clustered database
-    SmartRedis::Client client(cluster_mode);
+    SmartRedis::Client client(cluster_mode, __FILE__);
 
     // Put the tensor in the database
     std::string key = "3d_tensor";

examples/serial/fortran/smartredis_dataset.F90

@@ -75,7 +75,7 @@ program main
   if (result .ne. SRNoError) error stop 'dataset%get_meta_scalars failed'
 
   ! Initialize a client
-  result = client%initialize(.true.) ! Change .false. to .true. if not using a clustered database
+  result = client%initialize(.true., "smartredis_dataset") ! Change .false. to .true. if not using a clustered database
   if (result .ne. SRNoError) error stop 'client%initialize failed'
 
   ! Send the dataset to the database via the client

examples/serial/fortran/smartredis_put_get_3D.F90

@@ -46,7 +46,7 @@ program main
   call random_number(send_array_real_64)
 
   ! Initialize a client
-  result = client%initialize(.true.) ! Change .false. to .true. if not using a clustered database
+  result = client%initialize(.true., "smartredis_put_get_3D") ! Change .false. to .true. if not using a clustered database
   if (result .ne. SRNoError) error stop 'client%initialize failed'
 
   ! Send a tensor to the database via the client and verify that we can retrieve it

examples/serial/python/example_model_file_torch.py

@@ -43,7 +43,7 @@ def forward(self, x):
 
 # Connect a SmartRedis client
 db_address = "127.0.0.1:6379"
-client = Client(address=db_address, cluster=True)
+client = Client(address=db_address, cluster=True, logger_name="example_model_file_torch.py")
 
 try:
     net = Net()

examples/serial/python/example_model_torch.py

@@ -54,7 +54,7 @@ def forward(self, x):
 
 # Connect a SmartRedis client and set the model in the database
 db_address = "127.0.0.1:6379"
-client = Client(address=db_address, cluster=True)
+client = Client(address=db_address, cluster=True, logger_name="example_model_torch.py")
 client.set_model("torch_cnn", model, "TORCH", "CPU")
 
 # Retrieve the model and verify that the retrieved

examples/serial/python/example_put_get_dataset.py

@@ -39,7 +39,7 @@
 
 # Connect SmartRedis client to Redis database
 db_address = "127.0.0.1:6379"
-client = Client(address=db_address, cluster=True)
+client = Client(address=db_address, cluster=True, logger_name="example_put_get_dataset.py")
 
 # Place the DataSet into the database
 client.put_dataset(dataset)

examples/serial/python/example_put_get_tensor.py

@@ -29,7 +29,7 @@
 
 # Connect a SmartRedis client to Redis database
 db_address = "127.0.0.1:6379"
-client = Client(address=db_address, cluster=True)
+client = Client(address=db_address, cluster=True, logger_name="example_put_get_tensor.py")
 
 # Send a 2D tensor to the database
 key = "2D_array"

examples/serial/python/example_script.py

@@ -41,7 +41,7 @@ def two_to_one(data, data_2):
 
 # Connect a SmartRedis client to the Redis database
 db_address = "127.0.0.1:6379"
-client = Client(address=db_address, cluster=True)
+client = Client(address=db_address, cluster=True, logger_name="example_script.py")
 
 # Generate some test data to feed to the two_to_one function
 data = np.array([[1, 2, 3, 4]])

examples/serial/python/example_script_file.py

@@ -33,7 +33,7 @@
 
 # Connect to the Redis database
 db_address = "127.0.0.1:6379"
-client = Client(address=db_address, cluster=True)
+client = Client(address=db_address, cluster=True, logger_name="example_script_file.py")
 
 # Place the script in the database
 client.set_script_from_file(

include/c_client.h

@@ -43,10 +43,16 @@ extern "C" {
 /*!
 *   \brief C-client constructor
 *   \param cluster Flag to indicate if a database cluster is being used
+*   \param logger_name Identifier for the current client
+*   \param logger_name_length Length in characters of the logger_name string
 *   \param new_client Receives the new client
 *   \return Returns SRNoError on success or an error code on failure
 */
-SRError SmartRedisCClient(bool cluster, void **new_client);
+SRError SmartRedisCClient(
+    bool cluster,
+    const char* logger_name,
+    const size_t logger_name_length,
+    void **new_client);
 
 /*!
 *   \brief C-client destructor