From b59d859ec791a5b92f622800357967ec460266db Mon Sep 17 00:00:00 2001
From: Adam Stewart <adam.stewart73@gmail.com>
Date: Tue, 17 Nov 2020 11:26:54 +1100
Subject: [PATCH 1/7] Added custom 404 and 500 templates

---
 templates/404.html | 31 +++++++++++++++++++++++++++++++
 templates/500.html | 30 ++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+)
 create mode 100644 templates/404.html
 create mode 100644 templates/500.html

diff --git a/templates/404.html b/templates/404.html
new file mode 100644
index 000000000..c6b0b7bc6
--- /dev/null
+++ b/templates/404.html
@@ -0,0 +1,31 @@
+{% extends "base.html" %}
+
+{% load static %}
+
+{% block head %}
+
+{% endblock head %}
+
+{% block content %}
+<!-- Begin Page Content -->
+<div class="container-fluid">
+
+  <!-- 404 Error Text -->
+  <div class="text-center">
+      <div class="error mx-auto" data-text="404">404</div>
+      <p class="lead text-gray-800 mb-5">Page Not Found</p>
+      <p class="text-gray-500 mb-0">It looks like you found a transient page...</p>
+      <p class="text-gray-500 mb-0">Please report this error if it is unexpected.</p>
+      <a href="{% url 'index' %}">&larr; Back to Home</a>
+  </div>
+
+</div>
+<!-- /.container-fluid -->
+
+{% endblock content %}
+
+{% block custom_page_scripts %}
+
+<!-- Chart scripts -->
+
+{% endblock custom_page_scripts %}
\ No newline at end of file
diff --git a/templates/500.html b/templates/500.html
new file mode 100644
index 000000000..a68c3f319
--- /dev/null
+++ b/templates/500.html
@@ -0,0 +1,30 @@
+{% extends "base.html" %}
+
+{% load static %}
+
+{% block head %}
+
+{% endblock head %}
+
+{% block content %}
+<!-- Begin Page Content -->
+<div class="container-fluid">
+
+  <!-- 404 Error Text -->
+  <div class="text-center">
+      <div class="error mx-auto" data-text="500">500</div>
+      <p class="lead text-gray-800 mb-5">Internal Server Error</p>
+      <p class="text-gray-500 mb-0">Please report this error if it is unexpected.</p>
+      <a href="{% url 'index' %}">&larr; Back to Home</a>
+  </div>
+
+</div>
+<!-- /.container-fluid -->
+
+{% endblock content %}
+
+{% block custom_page_scripts %}
+
+<!-- Chart scripts -->
+
+{% endblock custom_page_scripts %}
\ No newline at end of file

From dbf61e2b1eb7cdf06f763b8f742a876f2f17c75f Mon Sep 17 00:00:00 2001
From: Adam Stewart <adam.stewart73@gmail.com>
Date: Tue, 17 Nov 2020 11:38:24 +1100
Subject: [PATCH 2/7] Update CHANGELOG.md

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ca14dac25..bb548422c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### Added
 
+- Added custom 404.html and 500.html templates for error pages [#415](https://github.com/askap-vast/vast-pipeline/pull/415)
 - Added ability to export measurement_pairs.parqyet as an arrow file [#393](https://github.com/askap-vast/vast-pipeline/pull/393).
 - Added new fields to detail pages and source and measurement tables [#406](https://github.com/askap-vast/vast-pipeline/pull/406).
 - Added new fields to source query page (island flux ratio, min and max fluxes) [#406](https://github.com/askap-vast/vast-pipeline/pull/406).
@@ -108,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### List of PRs
 
+- [#415](https://github.com/askap-vast/vast-pipeline/pull/415) feat: Added custom 404 and 500 templates.
 - [#393](https://github.com/askap-vast/vast-pipeline/pull/393) feat: Added measurement_pairs arrow export.
 - [#406](https://github.com/askap-vast/vast-pipeline/pull/406) feat, model: Added island flux ratio columns.
 - [#402](https://github.com/askap-vast/vast-pipeline/pull/402) fix: Fixed rms value calculation in non-cluster forced extractions.

From 0d61c4a439404f8424d7a1fa72054fe2a5553986 Mon Sep 17 00:00:00 2001
From: Adam Stewart <adam.stewart73@gmail.com>
Date: Wed, 18 Nov 2020 13:50:12 +1100
Subject: [PATCH 3/7] Fixed NAXIS=3 background and rms images.

- Also changed default order of images table on pipeline detail page to the date.
---
 CHANGELOG.md                          |  2 ++
 vast_pipeline/pipeline/new_sources.py | 10 ++--------
 vast_pipeline/views.py                |  1 +
 3 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ca14dac25..1fb80058f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -57,6 +57,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### Changed
 
+- Images table on pipeline run detail page changed to order by datetime by default [#417](https://github.com/askap-vast/vast-pipeline/pull/417).
 - Changed config argument `CREATE_MEASUREMENTS_ARROW_FILE` -> `CREATE_MEASUREMENTS_ARROW_FILES` [#393](https://github.com/askap-vast/vast-pipeline/pull/393).
 - Naming of average flux query fields to account for other min max flux fields [#406](https://github.com/askap-vast/vast-pipeline/pull/406).
 - Expanded `README.md` to include `DjangoQ` and UI job scheduling information [#404](https://github.com/askap-vast/vast-pipeline/pull/404).
@@ -89,6 +90,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### Fixed
 
+- Fixed measuring rms values from selavy created NAXIS=3 FITS images [#417](https://github.com/askap-vast/vast-pipeline/pull/417).
 - Fixed rms value calculation in non-cluster forced extractions [#402](https://github.com/askap-vast/vast-pipeline/pull/402).
 - Increase request limit for gunicorn [#398](https://github.com/askap-vast/vast-pipeline/pull/398).
 - Fixed max source Vs metric to being an absolute value [#391](https://github.com/askap-vast/vast-pipeline/pull/391).
diff --git a/vast_pipeline/pipeline/new_sources.py b/vast_pipeline/pipeline/new_sources.py
index 4d63f1ce2..31448854e 100644
--- a/vast_pipeline/pipeline/new_sources.py
+++ b/vast_pipeline/pipeline/new_sources.py
@@ -94,13 +94,7 @@ def get_image_rms_measurements(
     with fits.open(image) as hdul:
         header = hdul[0].header
         wcs = WCS(header, naxis=2)
-
-        try:
-            # ASKAP tile images
-            data = hdul[0].data[0, 0, :, :]
-        except Exception as e:
-            # ASKAP SWarp images
-            data = hdul[0].data
+        data = hdul[0].data.squeeze()
 
     # Here we mimic the forced fits behaviour,
     # sources within 3 half BMAJ widths of the image
@@ -229,7 +223,7 @@ def parallel_get_rms_measurements(
             get_image_rms_measurements,
             edge_buffer=edge_buffer,
             meta=col_dtype
-        ).compute(num_workers=n_cpu, scheduler='processes')
+        ).compute(num_workers=n_cpu, scheduler='single-threaded')
     )
 
     df = df.merge(
diff --git a/vast_pipeline/views.py b/vast_pipeline/views.py
index 8c0d7207c..c185ee69c 100644
--- a/vast_pipeline/views.py
+++ b/vast_pipeline/views.py
@@ -414,6 +414,7 @@ def RunDetail(request, id):
             'Beam PA (deg)'
         ],
         'search': True,
+        'order': [1, 'asc']
     }
 
     meas_fields = [

From 8131efe93410a245cd938ecb936d9bda44d1feb5 Mon Sep 17 00:00:00 2001
From: Adam Stewart <adam.stewart73@gmail.com>
Date: Wed, 18 Nov 2020 14:02:40 +1100
Subject: [PATCH 4/7] Fixed new source no valid measurements behaviour

- Added returns and checks for when all the rms_measurements to fetch are either out of range or too near a NaN value.
---
 vast_pipeline/pipeline/new_sources.py | 30 ++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/vast_pipeline/pipeline/new_sources.py b/vast_pipeline/pipeline/new_sources.py
index 31448854e..7ea7bf517 100644
--- a/vast_pipeline/pipeline/new_sources.py
+++ b/vast_pipeline/pipeline/new_sources.py
@@ -138,6 +138,15 @@ def get_image_rms_measurements(
 
     group = group.loc[valid_indexes]
 
+    if group.empty:
+        # early return if all sources failed range check
+        logger.debug(
+            f'All sources out of range in new source rms measurement'
+            ' for image {image}.'
+        )
+        group['img_diff_true_rms'] = np.nan
+        return group
+
     # Now we also need to check proximity to NaN values
     # as forced fits may also drop these values
     coords = SkyCoord(
@@ -165,15 +174,20 @@ def get_image_rms_measurements(
 
     valid_indexes = group[nan_valid].index.values
 
-    rms_values = data[
-        array_coords[0][nan_valid],
-        array_coords[1][nan_valid]
-    ]
+    if np.any(nan_valid):
+        # only run if there are actual values to measure
+        rms_values = data[
+            array_coords[0][nan_valid],
+            array_coords[1][nan_valid]
+        ]
+
+        # not matched ones will be NaN.
+        group.loc[
+            valid_indexes, 'img_diff_true_rms'
+        ] = rms_values.astype(np.float64) * 1.e3
 
-    # not matched ones will be NaN.
-    group.loc[
-        valid_indexes, 'img_diff_true_rms'
-    ] = rms_values.astype(np.float64) * 1.e3
+    else:
+        group['img_diff_true_rms'] = np.nan
 
     return group
 

From 0b2301b013fc6c186f834120460e9e42cba311d7 Mon Sep 17 00:00:00 2001
From: Adam Stewart <adam.stewart73@gmail.com>
Date: Wed, 18 Nov 2020 14:04:45 +1100
Subject: [PATCH 5/7] Update CHANGELOG.md

---
 CHANGELOG.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1fb80058f..b298ba9f4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -90,6 +90,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### Fixed
 
+- Fixed new sources rms measurement returns when no measurements are valid [#417](https://github.com/askap-vast/vast-pipeline/pull/417).
 - Fixed measuring rms values from selavy created NAXIS=3 FITS images [#417](https://github.com/askap-vast/vast-pipeline/pull/417).
 - Fixed rms value calculation in non-cluster forced extractions [#402](https://github.com/askap-vast/vast-pipeline/pull/402).
 - Increase request limit for gunicorn [#398](https://github.com/askap-vast/vast-pipeline/pull/398).

From ffe7207327b8548b88b16a5eaabf79a1687aeda0 Mon Sep 17 00:00:00 2001
From: Adam Stewart <adam.stewart73@gmail.com>
Date: Wed, 18 Nov 2020 14:06:30 +1100
Subject: [PATCH 6/7] Fixed string format and revert to processes

---
 vast_pipeline/pipeline/new_sources.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/vast_pipeline/pipeline/new_sources.py b/vast_pipeline/pipeline/new_sources.py
index 7ea7bf517..507437fe5 100644
--- a/vast_pipeline/pipeline/new_sources.py
+++ b/vast_pipeline/pipeline/new_sources.py
@@ -141,8 +141,8 @@ def get_image_rms_measurements(
     if group.empty:
         # early return if all sources failed range check
         logger.debug(
-            f'All sources out of range in new source rms measurement'
-            ' for image {image}.'
+            'All sources out of range in new source rms measurement'
+            f' for image {image}.'
         )
         group['img_diff_true_rms'] = np.nan
         return group
@@ -237,7 +237,7 @@ def parallel_get_rms_measurements(
             get_image_rms_measurements,
             edge_buffer=edge_buffer,
             meta=col_dtype
-        ).compute(num_workers=n_cpu, scheduler='single-threaded')
+        ).compute(num_workers=n_cpu, scheduler='processes')
     )
 
     df = df.merge(

From 53ddc7699abcad85f52e228d8d4d3dd0d53b5a45 Mon Sep 17 00:00:00 2001
From: Adam Stewart <adam.stewart73@gmail.com>
Date: Thu, 19 Nov 2020 11:37:30 +1100
Subject: [PATCH 7/7] Remove not needed blocks

---
 templates/404.html | 10 ----------
 templates/500.html | 10 ----------
 2 files changed, 20 deletions(-)

diff --git a/templates/404.html b/templates/404.html
index c6b0b7bc6..11717bc41 100644
--- a/templates/404.html
+++ b/templates/404.html
@@ -2,10 +2,6 @@
 
 {% load static %}
 
-{% block head %}
-
-{% endblock head %}
-
 {% block content %}
 <!-- Begin Page Content -->
 <div class="container-fluid">
@@ -23,9 +19,3 @@
 <!-- /.container-fluid -->
 
 {% endblock content %}
-
-{% block custom_page_scripts %}
-
-<!-- Chart scripts -->
-
-{% endblock custom_page_scripts %}
\ No newline at end of file
diff --git a/templates/500.html b/templates/500.html
index a68c3f319..adbefad36 100644
--- a/templates/500.html
+++ b/templates/500.html
@@ -2,10 +2,6 @@
 
 {% load static %}
 
-{% block head %}
-
-{% endblock head %}
-
 {% block content %}
 <!-- Begin Page Content -->
 <div class="container-fluid">
@@ -22,9 +18,3 @@
 <!-- /.container-fluid -->
 
 {% endblock content %}
-
-{% block custom_page_scripts %}
-
-<!-- Chart scripts -->
-
-{% endblock custom_page_scripts %}
\ No newline at end of file