Merge branch 'master' into test-reorg-warnings

stan-dev · Dec 9, 2021 · 5a0851e · 5a0851e
2 parents 805d1c6 + a51dbaf
commit 5a0851e
Show file tree

Hide file tree

Showing 9 changed files with 1,727 additions and 782 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -53,7 +53,7 @@ pipeline {
                     retry(3) { checkout scm }
                     sh 'git clean -xffd'
 
-                    def stanMathSigs = ['test/integration/signatures/stan_math_sigs.t'].join(" ")
+                    def stanMathSigs = ['test/integration/signatures/stan_math_signatures.t'].join(" ")
                     skipExpressionTests = utils.verifyChanges(stanMathSigs)
 
                     def runTestPaths = ['src', 'test/integration/good', 'test/stancjs'].join(" ")

diff --git a/src/frontend/dune b/src/frontend/dune
@@ -32,17 +32,20 @@
 
 (menhir
  (modules parser)
- (flags :standard --table --strict --unused-tokens --fixed-exception))
+ (flags
+  :standard
+  --table
+  --strict
+  --unused-tokens
+  --fixed-exception
+  --require-aliases))
 
 (rule
- (with-stdout-to
-  parser_updated.messages
-  (run menhir %{dep:parser.mly} --update-errors %{dep:parser.messages})))
-
-(rule
- (with-stdout-to
-  parser_updated_trimmed.messages
-  (run %{dep:strip_redundant_parser_state.py} %{dep:parser_updated.messages})))
+ (targets parser_updated.messages)
+ (action
+  (with-stdout-to
+   %{targets}
+   (run menhir %{dep:parser.mly} --update-errors %{dep:parser.messages}))))
 
 (rule
  (targets parser_new.messages)
@@ -52,15 +55,24 @@
    (run menhir --list-errors %{dep:parser.mly}))))
 
 (rule
- (alias update_messages)
+ (targets parser_updated_trimmed.messages)
  (action
   (progn
+   (with-stdout-to
+    %{targets}
+    (run
+     %{dep:strip_redundant_parser_state.py}
+     %{dep:parser_updated.messages}))
    (run
     %{dep:add_missing_messages.py}
     %{dep:parser.mly}
     %{dep:parser_new.messages}
-    %{dep:parser_updated_trimmed.messages})
-   (diff %{dep:parser.messages} %{dep:parser_updated_trimmed.messages}))))
+    parser_updated_trimmed.messages))))
+
+(rule
+ (alias update_messages)
+ (action
+  (diff parser.messages parser_updated_trimmed.messages)))
 
 (rule
  (alias runtest)

diff --git a/src/frontend/parser.messages b/src/frontend/parser.messages
diff --git a/src/frontend/parser.mly b/src/frontend/parser.mly
@@ -28,36 +28,47 @@ let nest_unsized_array basic_type n =
 let (!=) = Stdlib.(!=)
 %}
 
-%token FUNCTIONBLOCK DATABLOCK TRANSFORMEDDATABLOCK PARAMETERSBLOCK
-       TRANSFORMEDPARAMETERSBLOCK MODELBLOCK GENERATEDQUANTITIESBLOCK
-%token LBRACE RBRACE LPAREN RPAREN LBRACK RBRACK LABRACK RABRACK COMMA SEMICOLON
-       BAR
-%token RETURN IF ELSE WHILE FOR IN BREAK CONTINUE PROFILE
-%token VOID INT REAL COMPLEX VECTOR ROWVECTOR ARRAY MATRIX ORDERED POSITIVEORDERED SIMPLEX
-       UNITVECTOR CHOLESKYFACTORCORR CHOLESKYFACTORCOV CORRMATRIX COVMATRIX
-%token LOWER UPPER OFFSET MULTIPLIER
-%token <string> INTNUMERAL
-%token <string> REALNUMERAL
-%token <string> IMAGNUMERAL
-%token <string> STRINGLITERAL
-%token <string> IDENTIFIER
-%token TARGET
-%token QMARK COLON BANG MINUS PLUS HAT ELTPOW TRANSPOSE TIMES DIVIDE MODULO IDIVIDE
-       LDIVIDE ELTTIMES ELTDIVIDE OR AND EQUALS NEQUALS LEQ GEQ TILDE
-%token ASSIGN PLUSASSIGN MINUSASSIGN TIMESASSIGN DIVIDEASSIGN
-   ELTDIVIDEASSIGN ELTTIMESASSIGN
-%token ARROWASSIGN INCREMENTLOGPROB GETLP (* all of these are deprecated *)
-%token PRINT REJECT
-%token TRUNCATE
-%token EOF
+(* Token definitions. The quoted strings are aliases, used in the examples generated in
+   parser.messages. They have no semantic meaning; see
+   http://gallium.inria.fr/~fpottier/menhir/manual.html#sec%3Atokens
+*)
+%token FUNCTIONBLOCK "functions" DATABLOCK "data"
+       TRANSFORMEDDATABLOCK "transformed data" PARAMETERSBLOCK "parameters"
+       TRANSFORMEDPARAMETERSBLOCK "transformed parameters" MODELBLOCK "model"
+       GENERATEDQUANTITIESBLOCK "generated quantities"
+%token LBRACE "{" RBRACE "}" LPAREN "(" RPAREN ")" LBRACK "[" RBRACK "]"
+       LABRACK "<" RABRACK ">" COMMA "," SEMICOLON ";" BAR "|"
+%token RETURN "return" IF "if" ELSE "else" WHILE "while" FOR "for" IN "in"
+       BREAK "break" CONTINUE "continue" PROFILE "profile"
+%token VOID "void" INT "int" REAL "real" COMPLEX "complex" VECTOR "vector"
+       ROWVECTOR "row_vector" ARRAY "array" MATRIX "matrix" ORDERED "ordered"
+       POSITIVEORDERED "positive_ordered" SIMPLEX "simplex" UNITVECTOR "unit_vector"
+       CHOLESKYFACTORCORR "cholesky_factor_corr" CHOLESKYFACTORCOV "cholesky_factor_cov"
+       CORRMATRIX "corr_matrix" COVMATRIX "cov_matrix"
+%token LOWER "lower" UPPER "upper" OFFSET "offset" MULTIPLIER "multiplier"
+%token <string> INTNUMERAL "24"
+%token <string> REALNUMERAL "3.1415"
+%token <string> IMAGNUMERAL "1i"
+%token <string> STRINGLITERAL "\"hello world\""
+%token <string> IDENTIFIER "foo"
+%token TARGET "target"
+%token QMARK "?" COLON ":" BANG "!" MINUS "-" PLUS "+" HAT "^" ELTPOW ".^" TRANSPOSE "'"
+       TIMES "*" DIVIDE "/" MODULO "%" IDIVIDE "%/%" LDIVIDE "\\" ELTTIMES ".*"
+       ELTDIVIDE "./" OR "||" AND "&&" EQUALS "==" NEQUALS "!=" LEQ "<=" GEQ ">=" TILDE "~"
+%token ASSIGN "=" PLUSASSIGN "+=" MINUSASSIGN "-=" TIMESASSIGN "*="
+       DIVIDEASSIGN "/=" ELTDIVIDEASSIGN "./=" ELTTIMESASSIGN ".*="
+%token ARROWASSIGN "<-" INCREMENTLOGPROB "increment_log_prob" GETLP "get_lp" (* all of these are deprecated *)
+%token PRINT "print" REJECT "reject"
+%token TRUNCATE "T"
+%token EOF ""
 
 (* UNREACHABLE tokens will never be produced by the lexer, so we can use them as
    "a thing that will never parse". This is useful in a few places. For example,
    when we the parser to differentiate between different failing states for
    error message purposes, we can partially accept one of them and then fail by
    requiring an UNREACHABLE token.
  *)
-%token UNREACHABLE
+%token UNREACHABLE "<<<<UNREACHABLE>>>"
 
 %right COMMA
 %right QMARK COLON
@@ -79,7 +90,6 @@ let (!=) = Stdlib.(!=)
 %start <Ast.untyped_program> program functions_only
 %%
 
-
 (* Grammar *)
 
 (* program *)
@@ -181,6 +191,9 @@ future_keyword:
 
 decl_identifier:
   | id=identifier { id }
+  | id=reserved_word { id }
+
+reserved_word:
   (* Keywords cannot be identifiers but
      semantic check produces a better error message. *)
   | FUNCTIONBLOCK { build_id "functions" $loc }

diff --git a/src/frontend/strip_redundant_parser_state.py b/src/frontend/strip_redundant_parser_state.py
@@ -2,14 +2,27 @@
 import re
 import sys
 
+
+matcher = re.compile(
+    r"((program|functions_only):.*\n)"
+    r"(##.*\n)"
+    r"(## Concrete.*\n)?"
+    r"(##.*\n)+"
+    r"((program|functions_only):.*\n)"
+)
+
+
 def strip_redundant_parser_states(s):
-    return re.sub("(program:.*\n)(##.*\n)+(program:.*\n)",
-                  lambda m: m.group(1) + m.group(3),
-                  s)
+    return matcher.sub(
+        lambda m: m.group(1) + m.group(4) + m.group(6),
+        s,
+    )
+
 
 def strip_lines(s):
     return "\n".join(map(lambda s: s.rstrip(), s.splitlines()))
 
+
 if __name__ == "__main__":
     with open(sys.argv[1]) as f:
         messages = f.read()

diff --git a/src/middle/Stan_math_signatures.ml b/src/middle/Stan_math_signatures.ml
@@ -1779,6 +1779,16 @@ let () =
     , ReturnType UReal
     , [UArray UInt; UReal; UArray UVector]
     , AoS ) ;
+  add_unqualified
+    ( "ordered_probit_lpmf"
+    , ReturnType UReal
+    , [UArray UInt; UVector; UVector]
+    , AoS ) ;
+  add_unqualified
+    ( "ordered_probit_lpmf"
+    , ReturnType UReal
+    , [UArray UInt; UVector; UArray UVector]
+    , AoS ) ;
   add_unqualified ("ordered_probit_rng", ReturnType UInt, [UReal; UVector], AoS) ;
   add_binary_vec_real_real "owens_t" AoS ;
   add_nullary "pi" ;

diff --git a/...te/ordered_probit/ordered_probit_log.stan → ...e/ordered_probit/ordered_probit_lpmf.stan b/...te/ordered_probit/ordered_probit_log.stan → ...e/ordered_probit/ordered_probit_lpmf.stan
@@ -10,6 +10,8 @@ transformed data {
   transformed_data_real = ordered_probit_lpmf(d_int | d_real, d_vector);
   transformed_data_real = ordered_probit_lpmf(d_int_array | d_vector, d_vector);
   transformed_data_real = ordered_probit_lpmf(d_int_array | d_vector, d_vector_array);
+  transformed_data_real = ordered_probit_lpmf(d_int_array | d_real, d_vector);
+  transformed_data_real = ordered_probit_lpmf(d_int_array | d_real, d_vector_array);
 }
 parameters {
   real p_real;
@@ -22,6 +24,8 @@ transformed parameters {
   transformed_param_real = ordered_probit_lpmf(d_int | d_real, d_vector);
   transformed_param_real = ordered_probit_lpmf(d_int_array | d_vector, d_vector);
   transformed_param_real = ordered_probit_lpmf(d_int_array | d_vector, d_vector_array);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | d_real, d_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | d_real, d_vector_array);
   transformed_param_real = ordered_probit_lpmf(d_int | p_real, d_vector);
   transformed_param_real = ordered_probit_lpmf(d_int_array | p_vector, d_vector);
   transformed_param_real = ordered_probit_lpmf(d_int_array | p_vector, d_vector_array);
@@ -31,6 +35,8 @@ transformed parameters {
   transformed_param_real = ordered_probit_lpmf(d_int | p_real, p_vector);
   transformed_param_real = ordered_probit_lpmf(d_int_array | p_vector, p_vector);
   transformed_param_real = ordered_probit_lpmf(d_int_array | p_vector, p_vector_array);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | p_real, p_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | p_real, p_vector_array);
 }
 model {
   y_p ~ normal(0, 1);

diff --git a/...od/function-signatures/distributions/multivariate/discrete/ordered_probit/pretty.expected b/...od/function-signatures/distributions/multivariate/discrete/ordered_probit/pretty.expected
@@ -1,20 +1,45 @@
-  $ ../../../../../../../../../install/default/bin/stanc --auto-format ordered_probit_log.stan
-Semantic error in 'ordered_probit_log.stan', line 11, column 26 to column 79:
-   -------------------------------------------------
-     9:    real transformed_data_real;
-    10:    transformed_data_real = ordered_probit_lpmf(d_int | d_real, d_vector);
-    11:    transformed_data_real = ordered_probit_lpmf(d_int_array | d_vector, d_vector);
-                                   ^
-    12:    transformed_data_real = ordered_probit_lpmf(d_int_array | d_vector, d_vector_array);
-    13:  }
-   -------------------------------------------------
+  $ ../../../../../../../../../install/default/bin/stanc --auto-format ordered_probit_lpmf.stan
+data {
+  int d_int;
+  real d_real;
+  array[d_int] int d_int_array;
+  vector[d_int] d_vector;
+  array[d_int] vector[d_int] d_vector_array;
+}
+transformed data {
+  real transformed_data_real;
+  transformed_data_real = ordered_probit_lpmf(d_int | d_real, d_vector);
+  transformed_data_real = ordered_probit_lpmf(d_int_array | d_vector, d_vector);
+  transformed_data_real = ordered_probit_lpmf(d_int_array | d_vector, d_vector_array);
+  transformed_data_real = ordered_probit_lpmf(d_int_array | d_real, d_vector);
+  transformed_data_real = ordered_probit_lpmf(d_int_array | d_real, d_vector_array);
+}
+parameters {
+  real p_real;
+  vector[d_int] p_vector;
+  array[d_int] vector[d_int] p_vector_array;
+  real y_p;
+}
+transformed parameters {
+  real transformed_param_real;
+  transformed_param_real = ordered_probit_lpmf(d_int | d_real, d_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | d_vector, d_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | d_vector, d_vector_array);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | d_real, d_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | d_real, d_vector_array);
+  transformed_param_real = ordered_probit_lpmf(d_int | p_real, d_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | p_vector, d_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | p_vector, d_vector_array);
+  transformed_param_real = ordered_probit_lpmf(d_int | d_real, p_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | d_vector, p_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | d_vector, p_vector_array);
+  transformed_param_real = ordered_probit_lpmf(d_int | p_real, p_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | p_vector, p_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | p_vector, p_vector_array);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | p_real, p_vector);
+  transformed_param_real = ordered_probit_lpmf(d_int_array | p_real, p_vector_array);
+}
+model {
+  y_p ~ normal(0, 1);
+}
 
-Ill-typed arguments supplied to function 'ordered_probit_lpmf':
-(array[] int, vector, vector)
-Available signatures:
-(array[] int, real, array[] vector) => real
-  The second argument must be real but got vector
-(array[] int, real, vector) => real
-  The second argument must be real but got vector
-(int, real, vector) => real
-  The first argument must be int but got array[] int
diff --git a/test/integration/signatures/stan_math_signatures.t b/test/integration/signatures/stan_math_signatures.t
@@ -15665,6 +15665,8 @@ Display all Stan math signatures exposed in the language
   ordered_probit_lpmf(int, real, vector) => real
   ordered_probit_lpmf(array[] int, real, vector) => real
   ordered_probit_lpmf(array[] int, real, array[] vector) => real
+  ordered_probit_lpmf(array[] int, vector, vector) => real
+  ordered_probit_lpmf(array[] int, vector, array[] vector) => real
   ordered_probit_rng(real, vector) => int
   owens_t(real, real) => real
   owens_t(real, vector) => vector