compute ideal source coverage with astropy xmatch

CHANGELOG.md

@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### Changed
 
+- Reduced the memory footprint for computing the ideal source coverages by sky regions [#555](https://github.com/askap-vast/vast-pipeline/pull/555).
 - Gulp will only read `webinterface/.env` if the required vars are undefined in the current environment [#548](https://github.com/askap-vast/vast-pipeline/pull/548).
 
 #### Fixed
@@ -24,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 #### List of PRs
 
+- [#555](https://github.com/askap-vast/vast-pipeline/pull/555): fix: compute ideal source coverage with astropy xmatch.
 - [#551](https://github.com/askap-vast/vast-pipeline/pull/551): feat: added processing spinner to source query table.
 - [#550](https://github.com/askap-vast/vast-pipeline/pull/550): fix: missing changelog entry
 - [#548](https://github.com/askap-vast/vast-pipeline/pull/548): fix: only read .env if required vars are undefined.

vast_pipeline/pipeline/utils.py

@@ -17,7 +17,6 @@
 from typing import List, Optional, Dict, Tuple
 from astropy.io import fits
 from astropy.coordinates import SkyCoord, Angle
-from astropy.io import fits
 from django.conf import settings
 from django.contrib.auth.models import User
 from psutil import cpu_count
@@ -29,7 +28,6 @@
 from vast_pipeline.models import (
     Band, Image, Run, SkyRegion
 )
-from vast_pipeline.image.utils import on_sky_sep
 
 
 logger = logging.getLogger(__name__)
@@ -978,36 +976,31 @@ def get_src_skyregion_merged_df(
 
     del sources_df
 
-    # create dataframe with all skyregions and sources combinations
-    src_skyrg_df = cross_join(
-        srcs_df.drop(['epoch_list', 'img_list'], axis=1).reset_index(),
-        skyreg_df.drop('skyreg_img_epoch_list', axis=1)
+    # crossmatch sources with sky regions up to the max sky region radius
+    skyreg_coords = SkyCoord(
+        ra=skyreg_df.centre_ra, dec=skyreg_df.centre_dec, unit="deg"
     )
-
-    skyreg_df = skyreg_df.drop(
-        ['centre_ra', 'centre_dec', 'xtr_radius'],
-        axis=1
-    ).set_index('id')
-
-    src_skyrg_df['sep'] = np.rad2deg(
-        on_sky_sep(
-            np.deg2rad(src_skyrg_df['wavg_ra'].values),
-            np.deg2rad(src_skyrg_df['centre_ra'].values),
-            np.deg2rad(src_skyrg_df['wavg_dec'].values),
-            np.deg2rad(src_skyrg_df['centre_dec'].values),
-        )
+    srcs_coords = SkyCoord(ra=srcs_df.wavg_ra, dec=srcs_df.wavg_dec, unit="deg")
+    skyreg_idx, srcs_idx, sep, _ = srcs_coords.search_around_sky(
+        skyreg_coords, skyreg_df.xtr_radius.max() * u.deg
     )
+    skyreg_df = skyreg_df.drop(columns=["centre_ra", "centre_dec"]).set_index("id")
 
     # select rows where separation is less than sky region radius
     # drop not more useful columns and groupby source id
     # compute list of images
     src_skyrg_df = (
-        src_skyrg_df.loc[
-            src_skyrg_df.sep < src_skyrg_df.xtr_radius,
-            ['source', 'id', 'sep']
-        ].merge(skyreg_df, left_on='id', right_index=True)
-        .drop('id', axis=1)
-        .explode('skyreg_img_epoch_list')
+        pd.DataFrame(
+            {
+                "source": srcs_df.iloc[srcs_idx].index,
+                "id": skyreg_df.iloc[skyreg_idx].index,
+                "sep": sep.to("deg").value,
+            }
+        )
+        .merge(skyreg_df, left_on="id", right_index=True)
+        .query("sep < xtr_radius")
+        .drop(columns=["id", "xtr_radius"])
+        .explode("skyreg_img_epoch_list")
     )
 
     del skyreg_df