[wip] enabled debugging

NikolasMarkou · Mar 28, 2022 · b7d4420 · b7d4420
1 parent f69f1e4
commit b7d4420
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 6 deletions.
diff --git a/bfcnn/configs/resnet_grayscale_gaussian_4x3_shared_5x5_bn_16x3x3.json b/bfcnn/configs/resnet_grayscale_gaussian_4x3_shared_5x5_bn_16x3x3.json
@@ -35,6 +35,7 @@
   },
   "train": {
     "epochs": 100,
+    "debugging": true,
     "total_steps": -1,
     "checkpoints_to_keep": 3,
     "checkpoint_every": 10000,

diff --git a/bfcnn/train_loop.py b/bfcnn/train_loop.py
@@ -79,12 +79,7 @@ def train_loop(
     optimizer, lr_schedule = \
         optimizer_builder(config=config["train"]["optimizer"])
 
-    # --- create the help variables
-    global_step = tf.Variable(
-        0, trainable=False, dtype=tf.dtypes.int64, name="global_step")
-    global_epoch = tf.Variable(
-        0, trainable=False, dtype=tf.dtypes.int64, name="global_epoch")
-    summary_writer = tf.summary.create_file_writer(model_dir)
+
 
     # --- get the train configuration
     train_config = config["train"]
@@ -121,6 +116,18 @@ def train_loop(
     use_discriminator = MODEL_DISCRIMINATE_STR in config
     model_discriminate = None
 
+    # --- enable debugging
+    if train_config.get("debugging", False):
+        tf.debugging.experimental.enable_dump_debug_info(
+            str(model_dir), tensor_debug_mode="FULL_HEALTH")
+
+    # --- create the help variables
+    global_step = tf.Variable(
+        0, trainable=False, dtype=tf.dtypes.int64, name="global_step")
+    global_epoch = tf.Variable(
+        0, trainable=False, dtype=tf.dtypes.int64, name="global_epoch")
+    summary_writer = tf.summary.create_file_writer(model_dir)
+
     # --- build the denoise model
     tf.summary.trace_on(graph=True, profiler=False)
     with summary_writer.as_default():