Upgrade: Add script to upgrade to v1.3 from v1.0, v1.1, v1.2

Pivotal Tracker: 57615140

Changes:
    - Added all changelist files to upgrade to v1.3
    - Updated script to new upgrade
    - Fixed issues around whitespace between arguments
    - Release notes for v1.3
    - Updated version numbers
    - Update PGXN versioning style to accomodate requirements

ReleaseNotes.txt

@@ -8,6 +8,42 @@ A complete list of changes for each release can be obtained by viewing the git
 commit history located at https://github.com/madlib/madlib/commits/master.
 
 Current list of bugs and issues can be found at http://jira.madlib.net.
+--------------------------------------------------------------------------------
+MADlib v1.3
+
+Release Date: 2013-October-03
+
+New Features:
+* Cox Proportional Hazards:
+    - Added stratification support for Cox PH models. Stratification is used as
+    shorthand for building a Cox model that allows for more than one stratum,
+    and hence, allows for more than one baseline hazard function.
+    Stratification provides two pieces of key, flexible functionality for the
+    end user of Cox models:
+        -- Allows a categorical variable Z to be appropriately accounted for in
+        the model without estimating its predictive impact on the response
+        variable.
+        -- Categorical variable Z is predictive/associated with the response
+        variable, but Z may not satisfy the proportional hazards assumption
+    - Added a new function (cox_zph) that tests the proportional hazards
+    assumption of a Cox model. This allows the user to build Cox models and then
+    verify the relevance of the model.
+* NULL Handling:
+    - Modified behavior of linear and logistic regression to 'omit' rows
+    containing NULL values for any of the dependent and independent variables.
+    The number of rows skipped is provided as part of the output table.
+
+Deprecated functions:
+    - Cox Proportional Hazard function has been renamed to 'coxph_train'.
+    Old function names ('cox_prop_hazards' and 'cox_prop_hazards_regr')
+    have been deprecated, and will be removed in the next major version update.
+    - The aggregate form of linear regression ('linregr') has been deprecated.
+    The stored-procedure form ('linregr_train') should be used instead.
+
+Bug Fixes:
+    - Fixed a memory leak in the Apriori algorithm.
+
+
 --------------------------------------------------------------------------------
 MADlib v1.2
 

deploy/PGXN/CMakeLists.txt

@@ -3,8 +3,10 @@
 # ------------------------------------------------------------------------------
 
 set(MADLIB_PGXN_RELEASE_NUMBER 1)
+# set(MADLIB_PGXN_VERSION_STR
+#    "${MADLIB_VERSION_MAJOR}.${MADLIB_VERSION_MINOR}.${MADLIB_VERSION_PATCH}release${MADLIB_PGXN_RELEASE_NUMBER}
 set(MADLIB_PGXN_VERSION_STR
-    "${MADLIB_VERSION_MAJOR}.${MADLIB_VERSION_MINOR}.${MADLIB_VERSION_PATCH}release${MADLIB_PGXN_RELEASE_NUMBER}")
+    "${MADLIB_VERSION_MAJOR}.${MADLIB_VERSION_MINOR}.${MADLIB_VERSION_PATCH}")
 set(MADLIB_PGXN_NAME "madlib-pgxn-${MADLIB_PGXN_VERSION_STR}")
 
 configure_file(META.json.in META.json)

deploy/gppkg/CMakeLists.txt

@@ -2,7 +2,7 @@
 # Packaging for Greenplum's gppkg
 # ------------------------------------------------------------------------------
 
-set(MADLIB_GPPKG_VERSION "1.7")
+set(MADLIB_GPPKG_VERSION "1.7.1")
 set(MADLIB_GPPKG_RELEASE_NUMBER 1)
 set(MADLIB_GPPKG_RPM_SOURCE_DIR
     "${CMAKE_BINARY_DIR}/_CPack_Packages/Linux/RPM/${CPACK_PACKAGE_FILE_NAME}"

deploy/postflight.sh

@@ -2,7 +2,7 @@
 
 # $0 - Script Path, $1 - Package Path, $2 - Target Location, and $3 - Target Volumn
 
-MADLIB_VERSION=1.2
+MADLIB_VERSION=1.3
 
 find /usr/local/madlib/bin -type d -exec cp -RPf {} /usr/local/madlib/old_bin \; 2>/dev/null
 find /usr/local/madlib/bin -depth -type d -exec rm -r {} \; 2>/dev/null

src/config/Version.yml

@@ -1 +1 @@
-version: 1.2
+version: 1.3

src/madpack/changelist.yaml

@@ -1,4 +1,4 @@
-# Changelist for MADlib version 1.1 to 1.2
+# Changelist for MADlib version 1.2 to 1.3
 
 # This file contains all changes that were introduced in a new version of
 # MADlib. This changelist is used by the upgrade script to detect what objects
@@ -9,11 +9,11 @@
 # file installed on the upgrade version. All other files (that don't have
 # updates), are cleaned up to remove object replacements
 new module:
-    arima:
-    arima_forecast:
 
 # Changes in the types (UDT) including removal and modification
 udt:
+    __logregr_result:
+    linregr_result:
 
 # List of the UDF changes that affect the user externally.  This includes change
 # in function name, change in argument order or argument types, and removal of
@@ -22,10 +22,36 @@ udt:
 # are user views dependent on this function, since the original function will
 # not be present in the upgraded version.
 udf:
+    # linear regression: 'num_processed' added in 'linregr_result'
+    - linregr_final:
+        rettype: schema_madlib.linregr_result
+        argument: schema_madlib.bytea8
+    - linregr_merge_states:
+        rettype: schema_madlib.bytea8
+        argument: schema_madlib.bytea8, schema_madlib.bytea8
+    - linregr_transition:
+        rettype: schema_madlib.bytea8
+        argument: schema_madlib.bytea8, double precision, double precision[]
+
+    # logistic regression: 'num_processed' added in '__logregr_result'
+    - __logregr_cg_result:
+        rettype: schema_madlib.__logregr_result
+        argument: double precision[]
+
+    - __logregr_irls_result:
+        rettype: schema_madlib.__logregr_result
+        argument: double precision[]
+
+    - __logregr_igd_result:
+        rettype: schema_madlib.__logregr_result
+        argument: double precision[]
 
 # Changes to aggregates (UDA) including removal and modification
 # Overloaded functions should be mentioned separately
 uda:
-
+    - linregr:
+        rettype: schema_madlib.linregr_result
+        argument: double precision, double precision[]
+
 # Cast operators (UDC) updated/added in v1.1
 udc:

src/madpack/changelist_1.0_1.2.yaml → src/madpack/changelist_1.0_1.3.yaml

@@ -1,4 +1,4 @@
-# Changelist for MADlib version 1.0 to 1.2
+# Changelist for MADlib version 1.0 to 1.3
 
 # This file contains all changes that were introduced in a new version of
 # MADlib. This changelist is used by the upgrade script to detect what objects
@@ -17,7 +17,8 @@ new module:
 
 # Changes in the types (UDT) including removal and modification
 udt:
-
+    __logregr_result:
+    linregr_result:
 # List of the UDF changes that affect the user externally.  This includes change
 # in function name, change in argument order or argument types, and removal of
 # the function. In each case, the original function is as good as removed and a
@@ -30,9 +31,36 @@ udf:
         rettype: schema_madlib.matrix_result
         argument: matrix_in text, matrix_out text
 
+    # linear regression: 'num_processed' added in 'linregr_result'
+    - linregr_final:
+        rettype: schema_madlib.linregr_result
+        argument: schema_madlib.bytea8
+    - linregr_merge_states:
+        rettype: schema_madlib.bytea8
+        argument: schema_madlib.bytea8, schema_madlib.bytea8
+    - linregr_transition:
+        rettype: schema_madlib.bytea8
+        argument: schema_madlib.bytea8, double precision, double precision[]
+
+    # logistic regression: 'num_processed' added in '__logregr_result'
+    - __logregr_cg_result:
+        rettype: schema_madlib.__logregr_result
+        argument: double precision[]
+
+    - __logregr_irls_result:
+        rettype: schema_madlib.__logregr_result
+        argument: double precision[]
+
+    - __logregr_igd_result:
+        rettype: schema_madlib.__logregr_result
+        argument: double precision[]
+
 # Changes to aggregates (UDA) including removal and modification
 # Overloaded functions should be mentioned separately
 uda:
-
+    - linregr:
+        rettype: schema_madlib.linregr_result
+        argument: double precision, double precision[]
+
 # Cast operators (UDC) updated/added in v1.1/v1.2
-udc:
+udc:

src/madpack/changelist_1.1_1.3.yaml

@@ -0,0 +1,61 @@
+
+# Changelist for MADlib version 1.1 to 1.3
+
+# This file contains all changes that were introduced in a new version of
+# MADlib. This changelist is used by the upgrade script to detect what objects
+# should be upgraded (while retaining all other objects from the previous version)
+
+# New modules (actually .sql_in files) added in upgrade version
+# For these files the sql_in code is retained as is with the functions in the
+# file installed on the upgrade version. All other files (that don't have
+# updates), are cleaned up to remove object replacements
+new module:
+    arima:
+    arima_forecast:
+
+
+# Changes in the types (UDT) including removal and modification
+udt:
+    __logregr_result:
+    linregr_result:
+
+# List of the UDF changes that affect the user externally.  This includes change
+# in function name, change in argument order or argument types, and removal of
+# the function. In each case, the original function is as good as removed and a
+# new function is created. In such cases, we should abort the upgrade if there
+# are user views dependent on this function, since the original function will
+# not be present in the upgraded version.
+udf:
+    # linear regression: 'num_processed' added in 'linregr_result'
+    - linregr_final:
+        rettype: schema_madlib.linregr_result
+        argument: schema_madlib.bytea8
+    - linregr_merge_states:
+        rettype: schema_madlib.bytea8
+        argument: schema_madlib.bytea8, schema_madlib.bytea8
+    - linregr_transition:
+        rettype: schema_madlib.bytea8
+        argument: schema_madlib.bytea8, double precision, double precision[]
+
+    # logistic regression: 'num_processed' added in '__logregr_result'
+    - __logregr_cg_result:
+        rettype: schema_madlib.__logregr_result
+        argument: double precision[]
+
+    - __logregr_irls_result:
+        rettype: schema_madlib.__logregr_result
+        argument: double precision[]
+
+    - __logregr_igd_result:
+        rettype: schema_madlib.__logregr_result
+        argument: double precision[]
+
+# Changes to aggregates (UDA) including removal and modification
+# Overloaded functions should be mentioned separately
+uda:
+    - linregr:
+        rettype: schema_madlib.linregr_result
+        argument: double precision, double precision[]
+
+# Cast operators (UDC) updated/added in v1.1
+udc: