Fix percentile bug

.circleci/config.yml

@@ -1,25 +1,5 @@
 version: 2.1
 
-parameters:
-   pkg_name:
-      type: string
-      default: "genutil"
-   repo_name:
-      type: string
-      default: "genutil"
-   last_stable:
-      type: string
-      default: "8.2"
-   user:
-      type: string
-      default: "cdat"
-   label:
-      type: string
-      default: "nightly"
-   env_name:
-      type: string
-      default: "test_genutil"
-
 aliases:
   - &setup_env
     name: setup_env
@@ -31,28 +11,24 @@ aliases:
           export PROJECT_DIR=/home/circleci/project/workdir/linux
        fi
        echo "export WORKDIR=$PROJECT_DIR/$PY_VER" >> $BASH_ENV
-       cat $BASH_ENV
        source $BASH_ENV
        mkdir -p $WORKDIR
 
   - &setup_miniconda
     name: setup_miniconda
     command: |
        source $BASH_ENV
-       mkdir -p $WORKDIR
        git clone https://github.com/CDAT/cdat.git $WORKDIR/cdat
        # install_miniconda.py installs miniconda3 under $WORKDIR/miniconda
        python $WORKDIR/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3'
 
-
   - &conda_rerender
     name: conda_rerender
     command: |
        source $BASH_ENV
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
        conda activate base
-       echo "make conda-rerender conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR last_stable=$LAST_STABLE branch=$CIRCLE_BRANCH"
-       make conda-rerender conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR last_stable=$LAST_STABLE branch=$CIRCLE_BRANCH
+       make conda-rerender workdir=$WORKDIR branch=$CIRCLE_BRANCH
 
   - &conda_build
     name: conda_build
@@ -62,7 +38,7 @@ aliases:
        conda activate base
        os=`uname`
        artifacts_dir="artifacts/artifacts.${os}.py_${PY_VER}"
-       make conda-build conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR artifact_dir=$PWD/$artifacts_dir build_version=$PY_VER
+       make conda-build workdir=$WORKDIR artifact_dir=$artifacts_dir build_version=$PY_VER
 
   - &setup_run_tests
     name: setup_run_tests
@@ -72,19 +48,15 @@ aliases:
        conda activate base
        export CONDA_PY_VER="python=$PY_VER"
        export LIBNETCDF_VER="libnetcdf=*=${LIBNETCDF}_*"
-
-       echo "make setup-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME extra_pkgs=\"$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS\""
-       make setup-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME extra_pkgs="$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS"
-
-       make conda-dump-env conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME artifact_dir=$PWD/spec_artifacts conda_env_filename=$CIRCLE_JOB
+       make setup-tests workdir=$WORKDIR extra_pkgs="$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS"
+       make conda-dump-env workdir=$WORKDIR artifact_dir=$PWD/spec_artifacts conda_env_filename=$CIRCLE_JOB
 
   - &run_tests
     name: run_tests
     command: |
        source $BASH_ENV
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
-       conda activate $ENV_NAME
-       make run-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME workdir=$WORKDIR
+       make run-tests workdir=$WORKDIR
     no_output_timeout: 10m
 
   - &conda_upload
@@ -93,8 +65,8 @@ aliases:
        source $BASH_ENV
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
        conda activate base
-       UPLOAD_OPTIONS="conda_upload_token=$CONDA_UPLOAD_TOKEN user=$USER label=$LABEL"
-       make conda-upload $UPLOAD_OPTIONS conda=$WORKDIR/miniconda/bin/conda artifact_dir="$PWD/artifacts/*/"
+       UPLOAD_OPTIONS="conda_upload_token=$CONDA_UPLOAD_TOKEN"
+       make conda-upload workdir=$WORKDIR $UPLOAD_OPTIONS artifact_dir="artifacts/*/"
 
 executors:
    linux:
@@ -113,9 +85,6 @@ jobs:
             type: string
       executor: << parameters.os >>
       environment:
-         PKG_NAME: << pipeline.parameters.pkg_name >>
-         REPO_NAME: << pipeline.parameters.repo_name >>
-         LAST_STABLE: << pipeline.parameters.last_stable >>
          PY_VER: << parameters.py_ver >>
       steps:
          - checkout
@@ -141,8 +110,6 @@ jobs:
             type: string
       executor: << parameters.os >>
       environment:
-         PKG_NAME: << pipeline.parameters.pkg_name >>
-         ENV_NAME: << pipeline.parameters.env_name >>
          PY_VER: << parameters.py_ver >>
          LIBNETCDF: << parameters.libnetcdf >>
       steps:
@@ -164,8 +131,6 @@ jobs:
       machine:
          image: circleci/classic:latest
       environment:
-         USER: << pipeline.parameters.user >>
-         LABEL: << pipeline.parameters.label >>
          PY_VER: "3.7"
       steps:
          - checkout

Lib/statistics.py

@@ -1668,6 +1668,10 @@ def geometricmean(x, axis=0, max_pct_missing=100.):
 
 
 def _percentiles(out, percent):
+    # change 1d to 2d array
+    if out.ndim == 1:
+        out = out.reshape((1,)+out.shape)
+
     if cdms2.isVariable(out):
         out = MV2.sort(out, axis=0).asma()
         ns = MV2.count(out, axis=0).asma()
@@ -1696,10 +1700,8 @@ def _percentiles(out, percent):
             pass
         Aii = numpy.where(numpy.equal(ns, 1), 100., tmp)
         ii = numpy.where(numpy.equal(ii, ns), ns - 1, ii)
-        if numpy.rank(ii) > 0:
+        if numpy.ndim(ii) > 0:
             ii = ii.astype(numpy.int)
-# tmp = (p-Ai)/(Aii-Ai)*array_indexing.extract(out,ii) + \
-# (Aii-p)/(Aii-Ai)*array_indexing.extract(out,i)
 
         tmp = (p - Ai) / (Aii - Ai) * arrayindexing.get(out, ii) + \
             (Aii - p) / (Aii - Ai) * arrayindexing.get(out, i)

Lib/stats_checker.py

@@ -4,14 +4,15 @@
 from .averager import __check_weightoptions
 
 
-class StatisticsError (Exception):
+class StatisticsError(Exception):
     def __init__(self, args=None):
         """Create an exception"""
         self.args = args
 
     def __str__(self):
         """Calculate the string representation"""
         return str(self.args)
+
     __repr__ = __str__
 
 
@@ -32,10 +33,10 @@ def __makeweights(x, w, axes):
     if not numpy.ma.isarray(w):
         # Ok Krishna returned a list of 1D arrays.... Let's put it together
         axs = x.getAxisList()
-        axes = cdms2.order2index(axs, axes)[:len(cdms2.orderparse(axes))]
+        axes = cdms2.order2index(axs, axes)[: len(cdms2.orderparse(axes))]
         endax = []
         for i in range(len(axes)):
-            if w[i] == 'unweighted':
+            if w[i] == "unweighted":
                 w[i] = numpy.ma.ones(len(axs[axes[i]]), dtype=x.dtype.char)
             if i == 0:
                 wo = w[i]
@@ -45,8 +46,8 @@ def __makeweights(x, w, axes):
                 endax.append(axs[axes[i]])
         w = cdms2.MV2.array(wo)
         w.setAxisList(endax)
-# else:
-# w.setAxisList(x.getAxisList())
+    # else:
+    # w.setAxisList(x.getAxisList())
     return w
 
 
@@ -64,29 +65,31 @@ def __checker(x, y, w, axes, smally=0):
         x = numpy.ma.array(x, copy=0)
     if not numpy.ma.isarray(y) and y is not None:
         y = numpy.ma.array(y, copy=0)
-    if not numpy.ma.isarray(
-            w) and w is not None and not isinstance(w, type('')):
-        if not isinstance(w[0], type('')):
+    if not numpy.ma.isarray(w) and w is not None and not isinstance(w, type("")):
+        if not isinstance(w[0], type("")):
             w = numpy.ma.array(w, copy=0)
         else:
             if not xismv:
                 raise StatisticsError(
-                    'Error if weights are a list then x must be an MV2 !!!')
+                    "Error if weights are a list then x must be an MV2 !!!"
+                )
             w = __makeweights(x, w, axes)
             wismv = 1
     elif w is not None:
         if not xismv:
             raise StatisticsError(
-                'Error if weights are a list then x must be an MV2 !!!')
+                "Error if weights are a list then x must be an MV2 !!!"
+            )
         w = __makeweights(x, w, axes)
         wismv = 1
 
     if xismv * yismv * wismv != 1:
         # We didn't pass all MV2s shapes have to match (unless None)
         if smally == 0:
             if x.shape != numpy.ma.shape(y) and y is not None:
-                raise StatisticsError('Error x and y shape do not match !' +
-                                      str(x.shape) + ',' + str(numpy.ma.shape(y)))
+                raise StatisticsError(
+                    "Error x and y shape do not match !" + str(x.shape) + "," + str(numpy.ma.shape(y))
+                )
         else:
             shy = list(y.shape)
             shy2 = y.shape
@@ -96,7 +99,9 @@ def __checker(x, y, w, axes, smally=0):
                 for i in axes:
                     myaxes.append(eval(i))
             elif isinstance(axes, int):
-                myaxes = [axes, ]
+                myaxes = [
+                    axes,
+                ]
             else:
                 myaxes = list(axes)
             for anaxis in myaxes[::-1]:
@@ -109,25 +114,27 @@ def __checker(x, y, w, axes, smally=0):
                     sh[i] = myaxes[i]
                 y = numpy.ma.transpose(y, sh)
             if x.shape != numpy.ma.shape(y) and y is not None:
-                raise StatisticsError('Error x and y shape do not match (y shouldbe 1D less than x) !' +
-                                      str(x.shape) + ',' + str(shy2) + ' Remember y must be 1D less than x')
+                err_msg = "Error x and y shape do not match (y shouldbe 1D less than x) !"
+                raise StatisticsError(
+                    err_msg + str(x.shape) + "," + str(shy2) + " Remember y must be 1D less than x"
+                )
         if x.shape != numpy.ma.shape(w) and w is not None:
-            raise StatisticsError('Error x and weights shape do not match !' +
-                                  str(x.shape) + ',' + str(numpy.ma.shape(w)) +
-                                  ' ATTENTION if you are trynig to pass a list of 1D arrays' +
-                                  'for each dim, then x must be an MV2 !!!')
+            msg1 = "Error x and weights shape do not match !"
+            msg2 = " ATTENTION if you are trying to pass a list of 1D arrays for each dim, then x must be an MV2 !!!"
+            raise StatisticsError(msg1 + str(x.shape) + "," + str(numpy.ma.shape(w)) + msg2)
         if not isinstance(axes, type([])):
             axes = cdms2.orderparse(str(axes))
         for i in axes:
             if len(x.shape) < i:
-                raise StatisticsError('Error you have ' + str(len(x.shape)) +
-                                      ' dimensions and try to work on dim:' + str(i))
+                err_msg = "Error you have " + str(len(x.shape)) + " dimensions and try to work on dim:" + str(i)
+                raise StatisticsError(err_msg)
     else:
         if y is not None:
             x, y = grower(x, y)
             if x.shape != y.shape:
                 raise StatisticsError(
-                    'Error x and y have different shapes' + str(x.shape) + ', ' + str(y.shape))
+                    "Error x and y have different shapes" + str(x.shape) + ", " + str(y.shape)
+                )
         ax = x.getAxisList()
         xorder = x.getOrder(ids=1)
         # Now grows w
@@ -136,11 +143,13 @@ def __checker(x, y, w, axes, smally=0):
             for o in worder:
                 if o not in xorder:
                     raise StatisticsError(
-                        'Error weights have a dimension that is neither in x or y:' + o)
+                        "Error weights have a dimension that is neither in x or y:" + o
+                    )
             x, w = grower(x, w)
             if x.shape != w.shape:
                 raise StatisticsError(
-                    'Error x and weights have different shapes' + str(x.shape) + ', ' + str(w.shape))
+                    "Error x and weights have different shapes" + str(x.shape) + ", " + str(w.shape)
+                )
         # Last thing convert the axes input to numbers
         if isinstance(axes, type(1)):
             axes = str(axes)
@@ -149,24 +158,21 @@ def __checker(x, y, w, axes, smally=0):
             naxes = len(axesparse)
             for i in range(naxes):
                 o = axesparse[i]
-                if isinstance(o, type('')):
+                if isinstance(o, type("")):
                     for j in range(len(xorder)):
                         if xorder[j] == o:
                             axesparse[i] = j
                     # Well it must be a name for x y t....
-                    if isinstance(axesparse[i], type('')):
+                    if isinstance(axesparse[i], type("")):
                         for j in range(len(x.shape)):
                             if o[1:-1] == x.getAxis(j).id:
                                 axesparse[i] = j
                     # Everything failed the axis id must be not existing in the
                     # slab...
-                    if isinstance(axesparse[i], type('')):
+                    if isinstance(axesparse[i], type("")):
                         raise StatisticsError(
-                            'Error axis id :' +
-                            o +
-                            ' not found in first slab: ' +
-                            x.getOrder(
-                                ids=1))
+                            "Error axis id :" + o + " not found in first slab: " + x.getOrder(ids=1)
+                        )
             axes = axesparse
     # Now we have array those shape match, and a nice list of axes let's keep
     # going
@@ -176,8 +182,12 @@ def __checker(x, y, w, axes, smally=0):
     xorder = list(range(len(x.shape)))
     forder = []
     for i in range(naxes):
-        forder.append(axes[i])
-        n0 = n0 * xsh[axes[i]]
+        a = axes[i]
+        forder.append(a)
+        try:
+            n0 = n0 * xsh[a]
+        except IndexError:
+            raise Exception("Axis {} is out of bounds for dimension {}".format(a, len(xsh)))
     fsh = [n0]
     ax2 = []
     for i in range(len(x.shape)):