From 79037e37f623c4c5462a4ed0ff3212a6714a026b Mon Sep 17 00:00:00 2001 From: Robert Moss Date: Wed, 28 Apr 2021 11:18:49 -0700 Subject: [PATCH] Figure placement and duplicate reference fixes --- chapters/cem_variants.tex | 95 +++++++----- chapters/episodic_ast.tex | 69 +++++---- chapters/pomdpstresstesting.tex | 4 +- chapters/weakness_rec.tex | 12 +- main.tex | 4 +- references/cem_variants.bib | 234 ------------------------------ references/episodic_ast.bib | 44 +----- references/pomdpstresstesting.bib | 31 +--- references/weakness_rec.bib | 97 +------------ 9 files changed, 118 insertions(+), 472 deletions(-) diff --git a/chapters/cem_variants.tex b/chapters/cem_variants.tex index 6a4d368..c956aa4 100644 --- a/chapters/cem_variants.tex +++ b/chapters/cem_variants.tex @@ -8,7 +8,7 @@ Through random sampling, the CE-method assumes that there are enough objective function evaluations to accurately represent the objective. This may not be a problem for simple applications, but can be an issue for computationally expensive objective functions. Another assumption is that the initial parameters of the input distribution are wide enough to cover the design space of interest. For the case with a multivariate Gaussian distribution, this corresponds to an appropriate mean and wide covariance. -In rare-event simulations with many local minima, the CE-method can fail to find a global minima especially with sparse objective function evaluations. +In rare-event simulations with many local minima, the CE-method can fail to find the global minimum especially with sparse objective function evaluations. This work aims to address the key assumptions of the CE-method. We introduce variants of the CE-method that use surrogate modeling to approximate the objective function, thus updating the belief of the underlying objective through estimation. @@ -24,7 +24,7 @@ \section{Related Work} \label{sec:cem_related_work} -The cross-entropy method is popular in the fields of operations research, machine learning, and optimization \cite{kochenderfer2015decision,Kochenderfer2019}. +The cross-entropy method is popular in the fields of operations research, machine learning, and optimization \cite{kochenderfer2015decision,kochenderfer2019algorithms}. The combination of the cross-entropy method, surrogate modeling, and mixture models has been explored in other work \cite{bardenet2010surrogating}. The work in \cite{bardenet2010surrogating} proposed an adaptive grid approach to accelerate Gaussian-process-based surrogate modeling using mixture models as the prior in the cross-entropy method. They showed that a mixture model performs better than a single Gaussian when the objective function is multimodal. Our work differs in that we augment the ``elite'' samples both by an approximate surrogate model and by a subroutine call to the CE-method using the learned surrogate model. @@ -106,7 +106,7 @@ \subsection{Cross-Entropy Method} \label{sec:cem_background_cem} The threshold $\gamma_k$ becomes smaller that its initial value, thus artificially making events \textit{less rare} under $\vec{X} \sim g(\vec{x}; \vec{\theta}_k)$. In practice, the CE-method algorithm requires the user to specify a number of \textit{elite} samples $m_\text{elite}$ which are used when fitting the new parameters for iteration $k^\prime$. -Conveniently, if our distribution $g$ belongs to the \textit{natural exponential family} then the optimal parameters can be found analytically \cite{Kochenderfer2019}. For a multivariate Gaussian distribution parameterized by $\vec{\mu}$ and $\mat{\Sigma}$, the optimal parameters for the next iteration $k^\prime$ correspond to the maximum likelihood estimate (MLE): +Conveniently, if our distribution $g$ belongs to the \textit{natural exponential family} then the optimal parameters can be found analytically \cite{kochenderfer2019algorithms}. For a multivariate Gaussian distribution parameterized by $\vec{\mu}$ and $\mat{\Sigma}$, the optimal parameters for the next iteration $k^\prime$ correspond to the maximum likelihood estimate (MLE): \begin{align*} \vec{\mu}_{k^\prime} &= \frac{1}{m_\text{elite}} \sum_{i=1}^{m_\text{elite}} \vec{x}_i\\ \vec{\Sigma}_{k^\prime} &= \frac{1}{m_\text{elite}} \sum_{i=1}^{m_\text{elite}} (\vec{x}_i - \vec{\mu}_{k^\prime})(\vec{x}_i - \vec{\mu}_{k^\prime})^\top @@ -125,10 +125,10 @@ \subsection{Cross-Entropy Method} \label{sec:cem_background_cem} \begin{algorithmic} \Function{CrossEntropyMethod}{}($S, g, m, m_\text{elite}, k_\text{max}$) \For {$k \in [1,\ldots,k_\text{max}]$} - \State $\mat{X} \sim g(\;\cdot\;; \vec{\theta}_k)$ where $\mat{X} \in \R^m$ - \State $\mat{Y} \leftarrow S(\vec{x})$ for $\vec{x} \in \mat{X}$ - \State $\e \leftarrow$ store top $m_\text{elite}$ from $\mat{Y}$ - \State $\vec{\theta}_{k^\prime} \leftarrow \textproc{Fit}(g(\;\cdot\;; \vec{\theta}_k), \e)$ + \State $\mat{X} \sim g(\;\cdot\;; \vec{\theta}_k)$ where $\mat{X} \in \R^{|g|\times m}$\algorithmiccomment{draw $m$ samples from $g$} + \State $\mat{Y} \leftarrow S(\vec{x})$ for $\vec{x} \in \mat{X}$ \algorithmiccomment{evaluate samples $\mat{X}$ using objective $S$} + \State $\e \leftarrow$ store top $m_\text{elite}$ from $\mat{Y}$ \algorithmiccomment{select elite samples output from objective} + \State $\vec{\theta}_{k^\prime} \leftarrow \textproc{Fit}(g(\;\cdot\;; \vec{\theta}_k), \e)$ \algorithmiccomment{re-fit distribution $g$ using elite samples} \EndFor \State \Return $g(\;\cdot\;; \vec{\theta}_{k_\text{max}})$ \EndFunction @@ -178,7 +178,7 @@ \subsection{Surrogate Models} Surrogate models are a popular approach and have been used to evaluate rare-event probabilities in computationally expensive systems \cite{li2010evaluation,li2011efficient}. The simplest example of a surrogate model is linear regression. In this work, we focus on the \textit{Gaussian process} surrogate model. -A Gaussian process (GP) is a distribution over functions that predicts the underlying objective function $S$ and captures the uncertainty of the prediction using a probability distribution \cite{Kochenderfer2019}. +A Gaussian process (GP) is a distribution over functions that predicts the underlying objective function $S$ and captures the uncertainty of the prediction using a probability distribution \cite{kochenderfer2019algorithms}. This means a GP can be sampled to generate random functions, which can then be fit to our given data $\mat{X}$. A Gaussian process is parameterized by a mean function $\m(\mat{X})$ and kernel function $\mat{K}(\mat{X},\mat{X})$, which captures the relationship between data points as covariance values. We denote a Gaussian process that produces estimates $\hat{\vec{y}}$ as: @@ -203,7 +203,7 @@ \subsection{Surrogate Models} \begin{align*} k(\vec{x},\vec{x}^\prime) = \sigma^2\exp\left(- \frac{(\vec{x} - \vec{x}^\prime)^\top(\vec{x} - \vec{x}^\prime)}{2\ell^2}\right) \end{align*} -We refer to \cite{Kochenderfer2019} for a detailed overview of Gaussian processes and different kernel functions. +We refer to \cite{kochenderfer2019algorithms} for a detailed overview of Gaussian processes and different kernel functions. @@ -306,7 +306,7 @@ \subsection{Cross-Entropy Mixture Method} \label{sec:cem_alg_ce_mixture} The CE-mixture algorithm is identical to the CE-surrogate algorithm, but calls a custom \smallcaps{Fit} function to fit a mixture model to the elite set $\bfE$. The input distribution $\M$ is cast to a mixture model using the subcomponent distributions $\m$ as the components of the mixture. We use the default uniform weighting for each mixture component. -The mixture model $\M$ is then fit using the expectation-maximization algorithm shown in \cref{alg:em}, and the resulting distribution is returned. +The mixture model $\M$ is then fit using the expectation-maximization algorithm, and the resulting distribution is returned. The idea is to use the distributions in $\m$ that are centered around each true-elite as the components of the casted mixture model. Therefore, we would expect better performance of the CE-mixture method when the objective function has many competing local minima. Results in \cref{sec:cem_results} aim to show this behavior. @@ -497,26 +497,45 @@ \subsubsection{Scheduling Experiments} \label{sec:cem_schedule_experiments} \subsection{Results and Analysis} \label{sec:cem_results} -\begin{figure}[!hb] - % \centering - \resizebox{0.9\columnwidth}{!}{\input{figures/cem_variants/experiment1a.tex}} - \caption{ - \label{fig:experiment_1a} - Average optimal value for experiment (1A) when the initial mean is centered at the global minimum and the covariance sufficiently covers the design space. - } -\end{figure} +% \begin{figure}[!hb] +% % \centering +% \resizebox{0.9\columnwidth}{!}{\input{figures/cem_variants/experiment1a.tex}} +% \caption{ +% \label{fig:experiment_1a} +% Average optimal value for experiment (1A) when the initial mean is centered at the global minimum and the covariance sufficiently covers the design space. +% } +% \end{figure} \Cref{fig:experiment_1a} shows the average value of the current optimal $\bar{b}_v$ for the three algorithms for experiment (1A). One standard deviation is plotted in the shaded region. Notice that the standard CE-method converges to a local minima before $k_\text{max}$ is reached. Both CE-surrogate method and CE-mixture stay below the standard CE-method curve, highlighting the mitigation of convergence to local minima. Minor differences can be seen between CE-surrogate and CE-mixture, differing slightly towards the tail in favor of CE-surrogate. -The average runtime of the algorithms along with the performance metrics are shown together for each experiment in \cref{tab:results}. +The average runtime of the algorithms along with the performance metrics are shown together for each experiment in \cref{tab:cem_results}. + +\begin{figure*}[ht] + \centering + \subfloat[Average optimal value for experiment (1A) when the initial mean is centered at the global minimum and the covariance sufficiently covers the design space.]{% + \resizebox{0.45\textwidth}{!}{\input{figures/cem_variants/experiment1a.tex}} + \label{fig:experiment_1a} + } + \hspace{2mm} + \subfloat[Average optimal value for experiment (1B) when the initial mean is far from the global minimum with a wide covariance.]{% + \resizebox{0.45\textwidth}{!}{\input{figures/cem_variants/experiment1b.tex}} + \label{fig:experiment_1b} + } + \hspace{2mm} + \subfloat[Average optimal value for experiment (1C) when we restrict the number of objective function calls.]{% + \resizebox{0.45\textwidth}{!}{\input{figures/cem_variants/experiment1c.tex}} + \label{fig:experiment_1c} + } + \caption{Cross-entropy method variant experiment results.}\label{fig:cem_experiments} +\end{figure*} \begin{table}[!ht] \centering - \caption{\label{tab:results} Experimental results.} + \caption{\label{tab:cem_results} Experimental results.} \begin{tabular}{cllll} % p{3cm} \toprule \textbf{Exper.} & \textbf{Algorithm} & \textbf{Runtime} & $\bar{b}_v$ & $\bar{b}_d$\\ @@ -543,20 +562,20 @@ \subsection{Results and Analysis} \label{sec:cem_results} \end{table} An apparent benefit of the standard CE-method is in its simplicity and speed. -As shown in \cref{tab:results}, the CE-method is the fastest approach by about 2-3 orders of magnitude compared to CE-surrogate and CE-mixture. +As shown in \cref{tab:cem_results}, the CE-method is the fastest approach by about 2-3 orders of magnitude compared to CE-surrogate and CE-mixture. The CE-mixture method is notably the slowest approach. Although the runtime is also based on the objective function being tested, recall that we are using the same number of true objective function calls in each algorithm, and the metrics we are concerned with in optimization are to minimize $\bar{b}_v$ and $\bar{b}_d$. We can see that the CE-surrogate method consistently out performs the other methods. Surprisingly, a uniform evaluation schedule performs the best even in the sparse scenario where the initial mean is far away from the global optimal. -\begin{figure}[!ht] - % \centering - \resizebox{0.9\columnwidth}{!}{\input{figures/cem_variants/experiment1b.tex}} - \caption{ - \label{fig:experiment_1b} - Average optimal value for experiment (1B) when the initial mean is far from the global minimum with a wide covariance. - } -\end{figure} +% \begin{figure}[!ht] +% % \centering +% \resizebox{0.9\columnwidth}{!}{\input{figures/cem_variants/experiment1b.tex}} +% \caption{ +% \label{fig:experiment_1b} +% Average optimal value for experiment (1B) when the initial mean is far from the global minimum with a wide covariance. +% } +% \end{figure} When the initial mean of the input distribution is placed far away from the global optimal, the CE-method tends to converge prematurely as shown in \cref{fig:experiment_1b}. This scenario is illustrated in \cref{fig:example_1b}. @@ -564,7 +583,7 @@ \subsection{Results and Analysis} \label{sec:cem_results} \begin{figure}[!h] \centering - \resizebox{0.7\columnwidth}{!}{\input{figures/cem_variants/example1b.pgf}} + \resizebox{0.6\columnwidth}{!}{\input{figures/cem_variants/example1b.pgf}} \caption{ \label{fig:example_1b} First iteration of the scenario in experiment (1B) where the initial distribution is far away form the global optimal. The red dots indicate the true-elites, the black dots with white outlines indicate the ``non-elites'' evaluated from the true objective function, and the white dots with black outlines indicate the samples evaluated using the surrogate model. @@ -573,14 +592,14 @@ \subsection{Results and Analysis} \label{sec:cem_results} -\begin{figure}[!ht] - % \centering - \resizebox{0.9\columnwidth}{!}{\input{figures/cem_variants/experiment1c.tex}} - \caption{ - \label{fig:experiment_1c} - Average optimal value for experiment (1C) when we restrict the number of objective function calls. - } -\end{figure} +% \begin{figure}[!ht] +% % \centering +% \resizebox{0.9\columnwidth}{!}{\input{figures/cem_variants/experiment1c.tex}} +% \caption{ +% \label{fig:experiment_1c} +% Average optimal value for experiment (1C) when we restrict the number of objective function calls. +% } +% \end{figure} Given the same centered mean as before, when we restrict the number of objective function calls even further to just 50 we see interesting behavior. diff --git a/chapters/episodic_ast.tex b/chapters/episodic_ast.tex index e54ef96..b2a2ff9 100644 --- a/chapters/episodic_ast.tex +++ b/chapters/episodic_ast.tex @@ -63,7 +63,7 @@ \section{Background} \label{sec:ast_background} This section describes background of the adaptive stress testing approach and flight management systems. \subsection{Adaptive Stress Testing} \label{sec:ast_background_ast} -Adaptive stress testing (AST) is a black-box approach to find rare failure events in cyber-physical systems \cite{ast_acasx,lee2018adaptive}. +Adaptive stress testing (AST) is a black-box approach to find rare failure events in cyber-physical systems \cite{ast_acasx,lee2020adaptive}. The AST problem is formulated as a Markov decision process (MDP) and can be solved using reinforcement learning algorithms to guide the search towards likely failure events. AST can also be formulated more generally as other sequential decision-making processes, such as a partially observable Markov decision process (POMDP) \cite{ast_acasx,ast_av,ast_traj_plan}. @@ -231,20 +231,22 @@ \subsection{Modified Monte Carlo Tree Search} \label{sec:ast_mcts} This way, we can reduce the number of external system executions but still provide the search with information during tree expansion, namely using back-propagated values of the transition probabilities and the miss distance from previously finished rollouts. Choosing to evaluate at the end of the rollout provides the SUT with an expanded set of waypoints which it evaluates once the rollout has reached its maximum depth. Generally for AST problem formulations, the discount factor $\gamma$ is set to $1$. -Algorithm \ref{alg:mcts-pw} is the entry point of MCTS-PW. +\Cref{alg:mcts-pw} is the entry point of MCTS-PW and \cref{alg:mcts-pw-simulate,alg:mcts-pw-rollout,alg:mcts-pw-action-widen,alg:mcts-pw-deterministic-state} detail the sub-routines. \input{algorithms/mcts-pw-algorithm.tex} -\input{algorithms/mcts-pw-algorithm-part2.tex} \section{Implementation} \label{sec:ast_implementation} -The AST implementation was written in the Julia programming language \cite{julia}. -% Julia is an open-source, high-level, dynamically-typed, scientific computing language that can achieve speeds comparable to C \cite{julia}. +The AST implementation was written in the Julia programming language \cite{bezanson2017julia}. +% Julia is an open-source, high-level, dynamically-typed, scientific computing language that can achieve speeds comparable to C \cite{bezanson2017julia}. Implementation of the simulation environment around the SUT was also written in Julia, but this section will focus on the algorithms required for AST and MCTS-PW. Modifications to MCTS were implemented and merged into the existing MCTS.jl\footnote{\url{https://github.com/JuliaPOMDP/MCTS.jl}} Julia package. +\input{algorithms/mcts-pw-algorithm-part2.tex} + + \subsection{Interface}\label{sec:ast_implementation_interface} To apply AST to a general black-box system, a user has to provide the interface defined in \Cref{tab:interface}. % @@ -282,7 +284,7 @@ \subsection{Interface}\label{sec:ast_implementation_interface} % TODO. POMDPStressTesting.jl transferred to github.com/sisl \subsection{Stress Testing Julia Framework} -We have implemented the AST interface written in Julia as part of a new package called POMDPStressTesting.jl.\footnote{\url{https://github.com/sisl/POMDPStressTesting.jl}} +We have implemented the AST interface written in Julia as part of a new package called POMDPStressTesting.jl\footnote{\url{https://github.com/sisl/POMDPStressTesting.jl}} (see \cref{cha:tooling}). This package is inspired by work originally done in the AdaptiveStressTesting.jl\footnote{\url{https://github.com/sisl/AdaptiveStressTesting.jl}} package, but POMDPStressTesting.jl adheres to the \texttt{MDP} interface defined by the POMDPs.jl\footnote{\url{https://github.com/JuliaPOMDP/POMDPs.jl}} package \cite{pomdps_jl}. Thus, POMDPStressTesting.jl fits into the POMDPs.jl ecosystem, which is why it can use the MCTS.jl package as an off-the-shelf solver. This design choice allows other Julia packages within the POMDPs.jl ecosystem to be used; this includes solvers, simulation tools, policies, and visualizations. @@ -476,7 +478,7 @@ \subsection{Results and Analysis} -The collected aggregate results are shown in \Cref{tab:results}. +The collected aggregate results are shown in \Cref{tab:ast_results}. AST finds failures with relative likelihood about an order of magnitude greater than that of direct Monte Carlo. The CEM approach finds a small number of failures with very low relative likelihood. This is because CEM is using importance sampling and after re-weighting the samples using the true distribution, we would expect to get these extremely small likelihood values. @@ -493,7 +495,7 @@ \subsection{Results and Analysis} \begin{table}[!ht] \centering - \caption{\label{tab:results} Experimental Results} + \caption{\label{tab:ast_results} Experimental Results} \begin{threeparttable} \begin{tabular}{@{}lrrrrr@{}} \toprule @@ -526,25 +528,40 @@ \subsection{Example Failure} These failures are encoded in input files than can be deterministically played back through the system. -\begin{figure}[t!] -\noindent\makebox[\columnwidth][c]{% -\begin{minipage}{1.25\columnwidth} -\centering -\resizebox{0.55\columnwidth}{!}{\input{figures/episodic_ast/dasc-failure.pgf}} -\end{minipage}} -\caption{Full trajectory of an example arc length failure originating from KSFO. Red diamonds indicate the input waypoints selected by MCTS and the yellow circles indicate the output lateral packets from the SUT. The red box shows where the failure occurs, shown in more detail in \Cref{fig:example_failure_zoomed}.} -\label{fig:example_failure} -\end{figure} +\begin{figure*}[!t] + \centering + \subfloat[Full trajectory of an example arc length failure originating from KSFO. Red diamonds indicate the input waypoints selected by MCTS and the yellow circles indicate the output lateral packets from the SUT. The red box shows where the failure occurs, shown in more detail in \Cref{fig:example_failure_zoomed}.]{% + \resizebox{0.47\textwidth}{!}{\input{figures/episodic_ast/dasc-failure.pgf}} + \label{fig:example_failure}} + \hspace{2mm} + \subfloat[Example arc length failure, zoomed in from \Cref{fig:example_failure}. Notice two almost identical red waypoint diamonds, which are separated by about 0.08 \si{nmi} or about 486 \si{ft} (zoom in further for more detail). The arc length failure is shown as the extending arc after the center yellow waypoint, which extends about 3 \si{nmi} past its intended end waypoint. This extension is the negative miss distance.]{% + \resizebox{0.47\textwidth}{!}{\input{figures/episodic_ast/dasc-failure-zoomed.pgf}} + \label{fig:example_failure_zoomed}} + \caption{ + \label{fig:example_failure_full} Example failure found by adaptive stress testing. + } +\end{figure*} -\begin{figure}[b!] -\noindent\makebox[\columnwidth][c]{% -\begin{minipage}{1.25\columnwidth} -\centering -\resizebox{0.55\columnwidth}{!}{\input{figures/episodic_ast/dasc-failure-zoomed.pgf}} -\end{minipage}} -\caption{Example arc length failure, zoomed in from \Cref{fig:example_failure}. Notice two almost identical red waypoint diamonds, which are separated by about 0.08 \si{nmi} or about 486 \si{ft} (zoom in further for more detail). The arc length failure is shown as the extending arc after the center yellow waypoint, which extends about 3 \si{nmi} past its intended end waypoint. This extension is the negative miss distance.} -\label{fig:example_failure_zoomed} -\end{figure} + +% \begin{figure}[t!] +% \noindent\makebox[\columnwidth][c]{% +% \begin{minipage}{1.25\columnwidth} +% \centering +% \resizebox{0.55\columnwidth}{!}{\input{figures/episodic_ast/dasc-failure.pgf}} +% \end{minipage}} +% \caption{Full trajectory of an example arc length failure originating from KSFO. Red diamonds indicate the input waypoints selected by MCTS and the yellow circles indicate the output lateral packets from the SUT. The red box shows where the failure occurs, shown in more detail in \Cref{fig:example_failure_zoomed}.} +% \label{fig:example_failure} +% \end{figure} + +% \begin{figure}[b!] +% \noindent\makebox[\columnwidth][c]{% +% \begin{minipage}{1.25\columnwidth} +% \centering +% \resizebox{0.55\columnwidth}{!}{\input{figures/episodic_ast/dasc-failure-zoomed.pgf}} +% \end{minipage}} +% \caption{Example arc length failure, zoomed in from \Cref{fig:example_failure}. Notice two almost identical red waypoint diamonds, which are separated by about 0.08 \si{nmi} or about 486 \si{ft} (zoom in further for more detail). The arc length failure is shown as the extending arc after the center yellow waypoint, which extends about 3 \si{nmi} past its intended end waypoint. This extension is the negative miss distance.} +% \label{fig:example_failure_zoomed} +% \end{figure} % \section{Future Work} % \label{sec:ast_future_work} diff --git a/chapters/pomdpstresstesting.tex b/chapters/pomdpstresstesting.tex index d19b1c5..9763f10 100644 --- a/chapters/pomdpstresstesting.tex +++ b/chapters/pomdpstresstesting.tex @@ -2,7 +2,7 @@ \section{POMDPStressTesting.jl Summary} \href{https://github.com/sisl/POMDPStressTesting.jl}{POMDPStressTesting.jl} is a package that uses reinforcement learning and stochastic optimization to find likely failures in black-box systems through a technique called adaptive stress testing \cite{ast}. Adaptive stress testing (AST) has been used to find failures in safety-critical systems such as aircraft collision avoidance systems \cite{ast_acasx}, flight management systems \cite{ast_fms}, and autonomous vehicles \cite{ast_av}. -The POMDPStressTesting.jl package is written in Julia \cite{julia} and is part of the wider POMDPs.jl ecosystem \cite{pomdps_jl}, which provides access to simulation tools, policies, visualizations, and---most importantly---solvers. +The POMDPStressTesting.jl package is written in Julia \cite{bezanson2017julia} and is part of the wider POMDPs.jl ecosystem \cite{pomdps_jl}, which provides access to simulation tools, policies, visualizations, and---most importantly---solvers. We provide different solver variants including online planning algorithms such as Monte Carlo tree search \cite{mcts} and deep reinforcement learning algorithms such as trust region policy optimization (TRPO) \cite{trpo} and proximal policy optimization (PPO) \cite{ppo}. Stochastic optimization solvers such as the cross-entropy method \cite{cem} are also available and random search is provided as a baseline. Additional solvers can easily be added by adhering to the POMDPs.jl interface. @@ -81,7 +81,7 @@ \section{CrossEntropyVariants.jl Summary} (𝐌, bestₓ, bestᵥ) = ce_surrogate(f, 𝐌) \end{lstlisting} -The performance of the surrogate models may be dependent on the underlying objective function. The default surrogate model is a Gaussian process with the squared exponential kernel function \cite{Kochenderfer2019}. To use radial basis functions instead of a Gaussian process, you can specify a basis keyword input: +The performance of the surrogate models may be dependent on the underlying objective function. The default surrogate model is a Gaussian process with the squared exponential kernel function \cite{kochenderfer2019algorithms}. To use radial basis functions instead of a Gaussian process, you can specify a basis keyword input: \begin{lstlisting}[language=Julia] f = paraboloid # objective function diff --git a/chapters/weakness_rec.tex b/chapters/weakness_rec.tex index 89359b6..3df2675 100644 --- a/chapters/weakness_rec.tex +++ b/chapters/weakness_rec.tex @@ -10,7 +10,7 @@ \subsection{Related Work} Current approaches to validate black-box systems focus their search over input disturbances to find failures \citep{corso2020survey}. -A black-box reinforcement learning approach known as adaptive stress testing (AST) has recently been used to find the most likely failures in aircraft collision avoidance systems \citep{lee2015adaptive, lee2018adaptive}, aircraft flight management systems \citep{moss2020adaptive}, and autonomous vehicles \citep{koren2018adaptive, koren2019efficient}. +A black-box reinforcement learning approach known as adaptive stress testing (AST) has recently been used to find the most likely failures in aircraft collision avoidance systems \citep{lee2015adaptive, lee2020adaptive}, aircraft flight management systems \citep{moss2020adaptive}, and autonomous vehicles \citep{koren2018adaptive, koren2019efficient}. An underlying assumption of the AST problem formulation is that the system under test can be modeled as a sequential decision making process with explicitly defined states. This assumption limits the application of AST for validation over a dataset and ultimately creates \textit{more} data to validate due to applying input disturbances. Other approaches use model-based clustering to efficiently sample large datasets, but rely on tuned initialization parameters for good performance \citep{wehrens2004model}. @@ -235,12 +235,12 @@ \section{Analysis and Discussion} \begin{figure*}[ht] \centering \subfloat[Confusion matrix.]{% - \resizebox{0.49\textwidth}{!}{\input{figures/weakness_rec/confusion_adversary.pgf}} + \resizebox{0.45\textwidth}{!}{\input{figures/weakness_rec/confusion_adversary.pgf}} \label{fig:confusion_adversary} } % \hspace{5mm} \subfloat[Failure likelihood prediction per pixel.]{% - \resizebox{0.49\textwidth}{!}{\input{figures/weakness_rec/failure-likelihood.pgf}} + \resizebox{0.45\textwidth}{!}{\input{figures/weakness_rec/failure-likelihood.pgf}} \label{fig:failure_likelihood} } \caption{Failure classifier performance and predictions.} @@ -298,16 +298,16 @@ \section{Analysis and Discussion} \centering \subfloat[Adversarial true positives.]{% \resizebox{0.45\textwidth}{!}{ - \centerline{\includegraphics[width=0.6\textwidth]{figures/weakness_rec/MNIST-true-positives.png}} + \centerline{\includegraphics[width=0.7\textwidth]{figures/weakness_rec/MNIST-true-positives.png}} } \label{fig:true_positives} } \hspace{5mm} \subfloat[Adversarial false negatives.]{% \resizebox{0.45\textwidth}{!}{ - \centerline{\includegraphics[width=0.6\textwidth]{figures/weakness_rec/MNIST-false-negatives.png}} + \centerline{\includegraphics[width=0.7\textwidth]{figures/weakness_rec/MNIST-false-negatives.png}} } - \label{fig:true_positives} + \label{fig:false_negatives} } \caption{MNIST failure predictions from the adversary.}\label{fig:predictions} \end{figure*} diff --git a/main.tex b/main.tex index 77f6055..17006f0 100644 --- a/main.tex +++ b/main.tex @@ -76,7 +76,7 @@ \chapter{Conclusions} \end{document} % TODO: -% [ ] combine references.bib +% [-] combine references.bib % [x] See Dorsa notes % [ ] See Mykel notes % [ ] Subsections for tooling: CrossEntropyVariants.jl and FailureRepresentation.jl @@ -91,7 +91,7 @@ \chapter{Conclusions} % [ ] Add introduction % [ ] Flow together body chapters % [ ] Remove redundancies between them -% [ ] Change conclusions/future work into discussion +% [x] Change conclusions/future work into discussion % - Two big things: % [ ] 1) intro chapter % - shortest intro chapter possible (concise) diff --git a/references/cem_variants.bib b/references/cem_variants.bib index 7b852b2..73502ae 100644 --- a/references/cem_variants.bib +++ b/references/cem_variants.bib @@ -1,219 +1,4 @@ -@ARTICLE{monte_carlo, - author = {{Bird}, G.~A.}, - title = "{Monte-Carlo simulation in an engineering context}", - journal = {Progress in Astronautics and Aeronautics}, - keywords = {Computational Fluid Dynamics, Computerized Simulation, Fluid Flow, Kinetic Theory, Mechanical Engineering, Monte Carlo Method, Computer Programs, Mathematical Models, Minicomputers, Molecular Collisions, Thermodynamics and Statistical Physics}, - year = 1981, - month = jan, - volume = {74}, - pages = {239-255}, - adsurl = {https://ui.adsabs.harvard.edu/abs/1981PrAA...74..239B}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} - - -@INPROCEEDINGS{ast_acasx, -author={R. {Lee} and M. J. {Kochenderfer} and O. J. {Mengshoel} and G. P. {Brat} and M. P. {Owen}}, -booktitle={2015 IEEE/AIAA 34th Digital Avionics Systems Conference (DASC)}, -title={Adaptive stress testing of airborne collision avoidance systems}, -year={2015}, -volume={}, -number={}, -pages={6C2-1-6C2-13}, -keywords={aircraft;collision avoidance;Monte Carlo methods;tree searching;adaptive stress testing;airborne collision avoidance systems;reinforcement learning formulation;Monte Carlo tree search;MCTS;aircraft collision avoidance system;mid-air collisions;Lead;Atmospheric modeling;Quality of service}, -doi={10.1109/DASC.2015.7311450}, -ISSN={2155-7209}, -month={Sep.},} - - -@INPROCEEDINGS{ast_av, -author={M. {Koren} and S. {Alsaif} and R. {Lee} and M. J. {Kochenderfer}}, -booktitle={2018 IEEE Intelligent Vehicles Symposium (IV)}, -title={Adaptive Stress Testing for Autonomous Vehicles}, -year={2018}, -volume={}, -number={}, -pages={1-7}, -keywords={decision making;learning (artificial intelligence);Markov processes;Monte Carlo methods;stochastic processes;tree searching;vehicles;autonomous vehicles;collision scenarios;Markov decision process;failure scenarios;Monte Carlo Tree Search;MCTS;Deep Reinforcement Learning solutions;DRL;Monte Carlo sampling;reinforcement learning algorithms;adaptive stress testing;stochastic element perturbation;Autonomous vehicles;Sensors;Testing;Markov processes;Roads;Stress;Monte Carlo methods}, -doi={10.1109/IVS.2018.8500400}, -ISSN={1931-0587}, -month={June},} - - -@inbook{dast_acasx, -author = {Ritchie Lee and Ole Mengshoel and Anshu Saksena and Ryan Gardner and Daniel Genin and Jeffrey Brush and Mykel J. Kochenderfer}, -title = {Differential Adaptive Stress Testing of Airborne Collision Avoidance Systems}, -booktitle = {2018 AIAA Modeling and Simulation Technologies Conference}, -chapter = {}, -pages = {}, -doi = {10.2514/6.2018-1923} -} - - -@incollection{ast_safety, - title={Adaptive Stress Testing of Safety-Critical Systems}, - author={Lee, Ritchie and Mengshoel, Ole J and Kochenderfer, Mykel J}, - booktitle={Safe, Autonomous and Intelligent Vehicles}, - pages={77--95}, - year={2019}, - publisher={Springer} -} - -@inproceedings{ast_traj_plan, - title={Adaptive Stress Testing of Trajectory Planning Systems}, - author={Lee, Ritchie and Puig-Navarro, Javier and Agogino, Adrian K and Giannakoupoulou, Dimitra and Mengshoel, Ole J and Kochenderfer, Mykel J and Allen, B Danette}, - booktitle={AIAA Scitech 2019 Forum}, - pages={1454}, - year={2019} -} - -@phdthesis{lee_thesis, - title={AdaStress: Adaptive Stress Testing and Interpretable Categorization for Safety-Critical Systems}, - author={Lee, Ritchie}, - year={2019}, - school={Carnegie Mellon University} -} - -@article{trustworthy_ai, - title={Efficient Black-box Assessment of Autonomous Vehicle Safety}, - author={Norden, Justin and O'Kelly, Matthew and Sinha, Aman}, - journal={arXiv preprint arXiv:1912.03618}, - year={2019} -} - -@INPROCEEDINGS{fms, -author={S. {Liden}}, -booktitle={AIAA/IEEE Digital Avionics Systems Conference. 13th DASC}, -title={The evolution of Flight Management Systems}, -year={1994}, -volume={}, -number={}, -pages={157-169}, -keywords={aircraft computers;decision support systems;aircraft displays;computerised navigation;flight management;commercial aircraft;avionic equipment;757/767 systems;747-400;MD-80;MD-90;A310/A300-A600;A320-A321;ARINC 739;A330/A340;Aircraft;Aerospace electronics;Crisis management;Computer displays;Cost function;Fuels;Software design;Navigation;Computer interfaces;Aerospace industry}, -doi={10.1109/DASC.1994.369487}, -ISSN={null}, -month={Oct},} - -@incollection{fms_workload, - title={Flight deck automation and pilot workload}, - author={Roscoe, Alan H}, - booktitle={Vigilance and performance in automatized systems/vigilance et performance de l’Homme dans les syst{\`e}mes automatis{\'e}s}, - pages={111--122}, - year={1989}, - publisher={Springer} -} - -@inproceedings{mcts_ucb, - title={Continuous upper confidence trees}, - author={Cou{\"e}toux, Adrien and Hoock, Jean-Baptiste and Sokolovska, Nataliya and Teytaud, Olivier and Bonnard, Nicolas}, - booktitle={International Conference on Learning and Intelligent Optimization}, - pages={433--445}, - year={2011}, - organization={Springer} -} - -@misc{mcts_wildfire, - title={A Comparison of Monte Carlo Tree Search and Mathematical Optimization for Large Scale Dynamic Resource Allocation}, - author={Dimitris Bertsimas and J. Daniel Griffith and Vishal Gupta and Mykel J. Kochenderfer and Velibor V. Mišić and Robert Moss}, - year={2014}, - eprint={1405.5498}, - archivePrefix={arXiv}, -} - -@article{julia, - title={Julia: A fresh approach to numerical computing}, - author={Bezanson, Jeff and Edelman, Alan and Karpinski, Stefan and Shah, Viral B}, - journal={SIAM review}, - volume={59}, - number={1}, - pages={65--98}, - year={2017}, - publisher={SIAM} -} - -@article{arch_comp, - title={ARCH-COMP 2019 category report: falsification}, - author={Ernst, Gidon and Arcaini, Paolo and Donze, Alexandre and Fainekos, Georgios and Mathesen, Logan and Pedrielli, Giulia and Yaghoubi, Shakiba and Yamagata, Yoriyuki and Zhang, Zhenya}, - journal={EPiC Series in Computing}, - volume={61}, - pages={129--140}, - year={2019}, - publisher={EasyChair} -} - - -@article{pomdps_jl, - author = {Maxim Egorov and Zachary N. Sunberg and Edward Balaban and Tim A. Wheeler and Jayesh K. Gupta and Mykel J. Kochenderfer}, - title = {{POMDP}s.jl: A Framework for Sequential Decision Making under Uncertainty}, - journal = {Journal of Machine Learning Research}, - year = {2017}, - volume = {18}, - number = {26}, - pages = {1-5} -} - -@article{vincenty, - title={Direct and inverse solutions of geodesics on the ellipsoid with application of nested equations}, - author={Vincenty, Thaddeus}, - journal={Survey review}, - volume={23}, - number={176}, - pages={88--93}, - year={1975}, - publisher={Taylor \& Francis} -} - -@inproceedings{bayes_opt, - title={Generating adversarial driving scenarios in high-fidelity simulators}, - author={Abeysirigoonawardena, Yasasa and Shkurti, Florian and Dudek, Gregory}, - booktitle={2019 International Conference on Robotics and Automation (ICRA)}, - pages={8271--8277}, - year={2019}, - organization={IEEE} -} - -@inproceedings{hill_climbing, - title={Multi-armed Bandits for Boolean Connectives in Hybrid System Falsification}, - author={Zhang, Zhenya and Hasuo, Ichiro and Arcaini, Paolo}, - booktitle={International Conference on Computer Aided Verification}, - pages={401--420}, - year={2019}, - organization={Springer} -} - -@article{rrts, - title={Rapidly-exploring random trees-based test generation for autonomous vehicles}, - author={Tuncali, Cumhur Erkan and Fainekos, Georgios}, - journal={arXiv preprint arXiv:1903.10629}, - year={2019} -} - -@inproceedings{bootstrap, - title={Analyzing neighborhoods of falsifying traces in cyber-physical systems}, - author={Diwakaran, Ram Das and Sankaranarayanan, Sriram and Trivedi, Ashutosh}, - booktitle={Proceedings of the 8th International Conference on Cyber-Physical Systems}, - pages={109--119}, - year={2017} -} - -@article{mc_limitations, - title={Improvements to and limitations of Latin hypercube sampling}, - author={Huntington, DE and Lyrintzis, CS}, - journal={Probabilistic engineering mechanics}, - volume={13}, - number={4}, - pages={245--253}, - year={1998}, - publisher={Elsevier} -} - -@book{rubinstein2004cross, - title={The Cross-Entropy Method: A Unified Approach to Combinatorial Optimization, {Monte-Carlo} Simulation and Machine Learning}, - author={Rubinstein, RY and Kroese, DP}, - year={2004}, - publisher={Springer} -} @article{de2005tutorial, title={A Tutorial on the Cross-Entropy Method}, @@ -226,13 +11,6 @@ @article{de2005tutorial publisher={Springer} } -@Book{Kochenderfer2019, - Title = {Algorithms for Optimization}, - Author = {Kochenderfer, Mykel J and Wheeler, Tim A}, - Publisher = {MIT Press}, - Year = {2019} -} - @book{kochenderfer2015decision, title={{Decision Making Under Uncertainty: Theory and Application}}, @@ -371,16 +149,4 @@ @article{wang2016cross pages={42--52}, year={2016}, publisher={Elsevier} -} - - -@article{rubinstein1999cross, - title={The Cross-Entropy Method for Combinatorial and Continuous Optimization}, - author={Rubinstein, Reuven}, - journal={Methodology and Computing in Applied Probability}, - volume={1}, - number={2}, - pages={127--190}, - year={1999}, - publisher={Springer} } \ No newline at end of file diff --git a/references/episodic_ast.bib b/references/episodic_ast.bib index f4f17b3..9f4ac7d 100644 --- a/references/episodic_ast.bib +++ b/references/episodic_ast.bib @@ -190,33 +190,20 @@ @ARTICLE{monte_carlo } + @INPROCEEDINGS{ast_acasx, -author={R. {Lee} and M. J. {Kochenderfer} and O. J. {Mengshoel} and G. P. {Brat} and M. P. {Owen}}, -booktitle={IEEE/AIAA Digital Avionics Systems Conference (DASC)}, +author={Ritchie Lee and Mykel J. Kochenderfer and Ole J. Mengshoel and Guillaume P. Brat and Michael P. Owen}, +booktitle={{IEEE/AIAA Digital Avionics Systems Conference (DASC)}}, title={Adaptive stress testing of airborne collision avoidance systems}, year={2015}, volume={}, number={}, -pages={6C2-1-6C2-13}, -keywords={aircraft;collision avoidance;Monte Carlo methods;tree searching;adaptive stress testing;airborne collision avoidance systems;reinforcement learning formulation;Monte Carlo tree search;MCTS;aircraft collision avoidance system;mid-air collisions;Lead;Atmospheric modeling;Quality of service}, -doi={10.1109/DASC.2015.7311450}, ISSN={2155-7209}, -month={9},} +month={9}, +doi={10.1109/DASC.2015.7311450} +} -@INPROCEEDINGS{ast_av, -author={M. {Koren} and S. {Alsaif} and R. {Lee} and M. J. {Kochenderfer}}, -booktitle={IEEE Intelligent Vehicles Symposium (IV)}, -title={Adaptive Stress Testing for Autonomous Vehicles}, -year={2018}, -volume={}, -number={}, -pages={1-7}, -keywords={decision making;learning (artificial intelligence);Markov processes;Monte Carlo methods;stochastic processes;tree searching;vehicles;autonomous vehicles;collision scenarios;Markov decision process;failure scenarios;Monte Carlo Tree Search;MCTS;Deep Reinforcement Learning solutions;DRL;Monte Carlo sampling;reinforcement learning algorithms;adaptive stress testing;stochastic element perturbation;Autonomous vehicles;Sensors;Testing;Markov processes;Roads;Stress;Monte Carlo methods}, -doi={10.1109/IVS.2018.8500400}, -ISSN={1931-0587}, -month={6},} - @inbook{dast_acasx, author = {Ritchie Lee and Ole Mengshoel and Anshu Saksena and Ryan Gardner and Daniel Genin and Jeffrey Brush and Mykel J. Kochenderfer}, @@ -306,16 +293,6 @@ @misc{mcts_wildfire archivePrefix={arXiv}, } -@article{julia, - title={Julia: A fresh approach to numerical computing}, - author={Bezanson, Jeff and Edelman, Alan and Karpinski, Stefan and Shah, Viral B}, - journal={SIAM Review}, - volume={59}, - number={1}, - pages={65--98}, - year={2017}, - publisher={SIAM} -} @article{arch_comp, title={{ARCH-COMP} 2019 category report: falsification}, @@ -499,12 +476,3 @@ @inproceedings{aerts2018temporal year={2018}, organization={IEEE} } - - -@misc{do178c, - title={{Software Considerations in Airborne Systems and Equipment Certification}}, - author={RTCA}, - howpublished={DO-178C}, - month={12}, - year={2011} -} \ No newline at end of file diff --git a/references/pomdpstresstesting.bib b/references/pomdpstresstesting.bib index e5031c9..bbd7a4c 100644 --- a/references/pomdpstresstesting.bib +++ b/references/pomdpstresstesting.bib @@ -1,4 +1,4 @@ -@article{julia, +@article{bezanson2017julia, title={Julia: A fresh approach to numerical computing}, author={Jeff Bezanson and Alan Edelman and Stefan Karpinski and Viral B. Shah}, journal={{SIAM Review}}, @@ -11,16 +11,6 @@ @article{julia } -@article{pomdps_jl, - author = {Maxim Egorov and Zachary N. Sunberg and Edward Balaban and Tim A. Wheeler and Jayesh K. Gupta and Mykel J. Kochenderfer}, - title = {{{POMDP}s.jl: A Framework for Sequential Decision Making under Uncertainty}}, - journal = {{Journal of Machine Learning Research}}, - year = {2017}, - volume = {18}, - number = {26}, - pages = {1-5} -} - @incollection{ast, title={{Adaptive Stress Testing of Safety-Critical Systems}}, author={Ritchie Lee and Ole J. Mengshoel and Mykel J. Kochenderfer}, @@ -31,18 +21,6 @@ @incollection{ast doi={10.1007/978-3-319-97301-2_5} } -@INPROCEEDINGS{ast_acasx, -author={Ritchie Lee and Mykel J. Kochenderfer and Ole J. Mengshoel and Guillaume P. Brat and Michael P. Owen}, -booktitle={{IEEE/AIAA Digital Avionics Systems Conference (DASC)}}, -title={Adaptive stress testing of airborne collision avoidance systems}, -year={2015}, -volume={}, -number={}, -ISSN={2155-7209}, -month={9}, -doi={10.1109/DASC.2015.7311450} -} - @INPROCEEDINGS{ast_fms, author={Robert J. Moss and Ritchie Lee and Nicholas Visser and Joachim Hochwarth and James G. Lopez and Mykel J. Kochenderfer}, @@ -135,13 +113,6 @@ @article{falstar doi={10.1109/TCAD.2018.2858463} } -@article{gym, - title={{OpenAI} gym}, - author={Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, John and Tang, Jie and Zaremba, Wojciech}, - journal={arXiv:1606.01540}, - year={2016} -} - @misc{do178c, title={{Software Considerations in Airborne Systems and Equipment Certification}}, author={RTCA}, diff --git a/references/weakness_rec.bib b/references/weakness_rec.bib index 8951e3f..2562424 100644 --- a/references/weakness_rec.bib +++ b/references/weakness_rec.bib @@ -120,12 +120,6 @@ @article{hochreiter1997long publisher={MIT Press} } -@book{sutton2018reinforcement, - title={Reinforcement Learning: An Introduction}, - author={Sutton, Richard S and Barto, Andrew G}, - year={2018}, - publisher={MIT press} -} @book{lavalle2006planning, title={Planning Algorithms}, @@ -583,16 +577,6 @@ @article{silver2017mastering year={2017} } -@article{silver2016mastering, - title={Mastering the game of Go with deep neural networks and tree search}, - author={Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others}, - journal={Nature}, - volume={529}, - number={7587}, - pages={484}, - year={2016}, - publisher={Nature Publishing Group} -} @inproceedings{delmas2019evaluation, title={An Evaluation of {M}onte {C}arlo Tree Search for Property Falsification on Hybrid Flight Control Laws}, @@ -631,14 +615,7 @@ @article{mullins2018adaptive publisher={Elsevier} } -@inproceedings{aerts2018temporal, - title={Temporal logic falsification of cyber-physical systems: An input-signal-space optimization approach}, - author={Aerts, Arend and Minh, Bryan Tong and Mousavi, Mohammad Reza and Reniers, Michel A}, - booktitle={IEEE International Conference on Software Testing, Verification and Validation Workshops (ICSTW)}, - pages={214--223}, - year={2018}, - organization={IEEE} -} + @inproceedings{pant2017smooth, title={Smooth operator: Control using the smooth robustness of temporal logic}, @@ -926,16 +903,6 @@ @article{huang2017accelerated publisher={IEEE} } -@article{de2005tutorial, - title={A tutorial on the cross-entropy method}, - author={De Boer, Pieter-Tjerk and Kroese, Dirk P and Mannor, Shie and Rubinstein, Reuven Y}, - journal={Annals of Operations Research}, - volume={134}, - number={1}, - pages={19--67}, - year={2005}, - publisher={Springer} -} @book{rubinstein2013cross, title={The Cross-Entropy Method: A Unified Approach to Combinatorial Optimization, {M}onte {C}arlo Simulation and Machine Learning}, @@ -962,16 +929,6 @@ @article{lee2015adaptive year={2015}, } -@article{abbas2013probabilistic, - title={Probabilistic temporal logic falsification of cyber-physical systems}, - author={Abbas, Houssam and Fainekos, Georgios and Sankaranarayanan, Sriram and Ivan{\v{c}}i{\'c}, Franjo and Gupta, Aarti}, - journal={ACM Transactions on Embedded Computing Systems (TECS)}, - volume={12}, - number={2s}, - pages={1--30}, - year={2013}, - publisher={ACM New York, NY, USA} -} @article{behzadan2019adversarial, title={Adversarial reinforcement learning framework for benchmarking collision avoidance mechanisms in autonomous vehicles}, @@ -1015,12 +972,6 @@ @book{baier2008principles publisher={MIT Press} } -@book{kochenderfer2019algorithms, - title={Algorithms for Optimization}, - author={Kochenderfer, Mykel J and Wheeler, Tim A}, - year={2019}, - publisher={MIT Press} -} @article{corso2020scalable, title={Scalable Autonomous Vehicle Safety Validation through Dynamic Programming and Scene Decomposition}, @@ -1089,13 +1040,6 @@ @Article{Silver2016 publisher={Nature Research} } -@Book{dmubook, - author = {Mykel J. Kochenderfer}, - title = {Decision Making Under Uncertainty: Theory and Application}, - year = {2015}, - publisher = {MIT Press}, -} - @inproceedings{He2017, title={Mask {R-CNN}}, author={He, Kaiming and Gkioxari, Georgia and Doll{\'a}r, Piotr and Girshick, Ross}, @@ -1152,12 +1096,6 @@ @Book{Russell2020 Edition= {4} } -@book{Sutton2018, - title={Reinforcement Learning: An Introduction}, - author={Sutton, Richard S and Barto, Andrew G}, - year={2018}, - publisher={MIT {P}ress} -} @inproceedings{Pillai2019, title={Superdepth: Self-Supervised, Super-Resolved Monocular Depth Estimation}, @@ -1280,24 +1218,6 @@ @article{salimans2018learning year={2018} } -@article{brockman2016openai, - title={Open{AI} gym}, - author={Brockman, Greg and Cheung, Vicki and Pettersson, Ludwig and Schneider, Jonas and Schulman, John and Tang, Jie and Zaremba, Wojciech}, - journal=arxiv, - number={1606.01540}, - year={2016} -} - -@article{bezanson2017julia, - title={Julia: A fresh approach to numerical computing}, - author={Bezanson, Jeff and Edelman, Alan and Karpinski, Stefan and Shah, Viral B}, - journal={SIAM Review}, - volume={59}, - number={1}, - pages={65--98}, - year={2017}, - publisher={SIAM} -} @article{yeh2018autonomous, title={Autonomous Systems and the Challenges in Verification, Validation, and Test}, @@ -1445,21 +1365,6 @@ @inproceedings{lee2018interpretable organization={SIAM} } -@misc{lee2018adaptive, - title={Adaptive Stress Testing: Finding Likely Failure Events with Reinforcement Learning}, - author={Ritchie Lee and Ole J. Mengshoel and Anshu Saksena and Ryan Gardner and Daniel Genin and Joshua Silbermann and Michael Owen and Mykel J. Kochenderfer}, - year={2018}, - eprint={1811.02188}, - archivePrefix={arXiv}, -} - -@misc{corso2020survey, - title={A Survey of Algorithms for Black-Box Safety Validation}, - author={Anthony Corso and Robert J. Moss and Mark Koren and Ritchie Lee and Mykel J. Kochenderfer}, - year={2020}, - eprint={2005.02979}, - archivePrefix={arXiv} -} @article{huang2005sampling, title={Sampling for {Bayesian} computation with large datasets},