Skip to content

Commit

Permalink
Fixed the evaluation count
Browse files Browse the repository at this point in the history
  • Loading branch information
MasWag committed Apr 20, 2024
1 parent abb1574 commit c819f53
Show file tree
Hide file tree
Showing 7 changed files with 203 additions and 115 deletions.
6 changes: 3 additions & 3 deletions core/src/main/java/net/maswag/BlackBoxVerifier.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
*/
public class BlackBoxVerifier<I> {
private static final Function<String, String> EDGE_PARSER = s -> s;
final private double multiplier = 1.0;
@Getter
MembershipOracle.MealyMembershipOracle<String, String> memOracle;
private final SUL<String, String> verifiedSystem;
Expand Down Expand Up @@ -110,7 +109,7 @@ public void addCompleteExplorationEQOracle(int minDepth, int maxDepth, int batch
* @param length The length of the input.
* @param minStep The minimum step of the corner case.
*/
public void addCornerCaseEQOracle(int length, int minStep) {
public EvaluationCountable. MealyEquivalenceOracle<String, String> addCornerCaseEQOracle(int length, int minStep) {
MealyFixedSetEQOracle cornerCaseEqOracle =
new MealyFixedSetEQOracle(this.getProperties().list(), this.memOracle);
int step = length;
Expand All @@ -129,10 +128,11 @@ public void addCornerCaseEQOracle(int length, int minStep) {
}
// Limit the length of each corner cases and add them to the oracle
cornerCase.stream().map(word -> word.prefix(length)).forEach(cornerCaseEqOracle::add);
//cornerCaseEqOracle.addAll(cornerCase.stream().map(word -> word.prefix(length)).collect(Collectors.toList()));
step /= 2;
}
addEqOracle(cornerCaseEqOracle);

return cornerCaseEqOracle;
}

public void addEqOracle(PropertyOracle.MealyEquivalenceOracle<String, String> eqOracle) {
Expand Down
11 changes: 7 additions & 4 deletions core/src/main/java/net/maswag/MealyFixedSetEQOracle.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import de.learnlib.oracle.MembershipOracle;
import de.learnlib.oracle.PropertyOracle;
import de.learnlib.query.DefaultQuery;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import net.automatalib.automaton.transducer.MealyMachine;
import net.automatalib.word.Word;
Expand All @@ -19,11 +20,12 @@
* Equivalence oracle that uses a fixed set of samples.
*/
@Slf4j
public class MealyFixedSetEQOracle implements EquivalenceOracle.MealyEquivalenceOracle<String, String> {
public class MealyFixedSetEQOracle implements EquivalenceOracle.MealyEquivalenceOracle<String, String>, EvaluationCountable. MealyEquivalenceOracle<String, String> {
List<Word<String>> fixedSamples = new ArrayList<>();
private final List<PropertyOracle.MealyPropertyOracle<String, String, String>> ltlOracles;
protected final MembershipOracle.MealyMembershipOracle<String, String> memOracle;

@Getter
protected int evaluateCount = 0;

/**
* Constructor for MealyFixedSetEQOracle.
Expand All @@ -37,11 +39,11 @@ public MealyFixedSetEQOracle(List<PropertyOracle.MealyPropertyOracle<String, Str
this.memOracle = memOracle;
}

void add(Word<String> sample) {
public void add(Word<String> sample) {
fixedSamples.add(sample);
}

void addAll(Collection<Word<String>> samples) {
public void addAll(Collection<Word<String>> samples) {
fixedSamples.addAll(samples);
}

Expand Down Expand Up @@ -75,6 +77,7 @@ public DefaultQuery<String, Word<String>> findCounterExample(MealyMachine<?, Str
for (Word<String> sample : fixedSamples) {
DefaultQuery<String, Word<String>> query = new DefaultQuery<>(sample);
memOracle.processQuery(query);
evaluateCount++;
Word<String> hypOutput = hypothesis.computeOutput(query.getInput());
log.trace("Processing fixed sample: {}", sample);
log.trace("Hypothesis output: {}", hypOutput);
Expand Down
13 changes: 6 additions & 7 deletions core/src/main/java/net/maswag/NumericSULVerifier.java
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,8 @@ public class NumericSULVerifier {
* @param signalStep The signal step in the simulation
* @param properties The LTL properties to be verified
* @param mapper The I/O mapepr between abstract/concrete Simulink models.
* @throws java.lang.Exception It can be thrown from the constructor of SimulinkSUL.
*/
public NumericSULVerifier(NumericSUL rawSUL, double signalStep, AdaptiveSTLUpdater<List<Double>> properties, NumericSULMapper mapper) throws Exception {
public NumericSULVerifier(NumericSUL rawSUL, double signalStep, AdaptiveSTLUpdater<List<Double>> properties, NumericSULMapper mapper) {
this.rawSUL = rawSUL;
this.signalStep = signalStep;
this.mapper = mapper;
Expand Down Expand Up @@ -80,19 +79,19 @@ public void addWpMethodEQOracle(int maxDepth) {
this.verifier.addWpMethodEQOracle(maxDepth);
}

void addBFOracle(double multiplier) {
public void addBFOracle(double multiplier) {
this.verifier.addBFOracle(multiplier);
}

void addRandomWordEQOracle(int minLength, int maxLength, int maxTests, Random random, int batchSize) {
public void addRandomWordEQOracle(int minLength, int maxLength, int maxTests, Random random, int batchSize) {
this.verifier.addRandomWordEQOracle(minLength, maxLength, maxTests, random, batchSize);
}

void addRandomWalkEQOracle(double restartProbability, long maxSteps, Random random) {
public void addRandomWalkEQOracle(double restartProbability, long maxSteps, Random random) {
this.verifier.addRandomWalkEQOracle(restartProbability, maxSteps, random);
}

void addCompleteExplorationEQOracle(int minDepth, int maxDepth, int batchSize) {
public void addCompleteExplorationEQOracle(int minDepth, int maxDepth, int batchSize) {
this.verifier.addCompleteExplorationEQOracle(minDepth, maxDepth, batchSize);
}

Expand All @@ -103,7 +102,7 @@ void addCompleteExplorationEQOracle(int minDepth, int maxDepth, int batchSize) {
* @param minStep The minimum step of the corner case.
*/
public void addCornerCaseEQOracle(int length, int minStep) {
this.verifier.addCornerCaseEQOracle(length, minStep);
this.evaluationCountables.add(this.verifier.addCornerCaseEQOracle(length, minStep));
}

/**
Expand Down
130 changes: 49 additions & 81 deletions example/kotlin/ATS1-step-5.ipynb

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions example/kotlin/ATS6a.main.kts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ SimulinkSUL(initScript, paramNames, signalStep, simulinkSimulationStep).use { au
val verifier = NumericSULVerifier(autoTransSUL, signalStep, properties, mapper)
// Timeout must be set before adding equivalence testing
verifier.setTimeout(50 * 60) // 50 minutes
// We first try equivalence testing based on corner case inputs
verifier.addCornerCaseEQOracle(signalLength, signalLength / 2)
// Then, we use genetic algorithm
verifier.addGAEQOracleAll(
signalLength,
maxTest,
Expand All @@ -96,4 +99,7 @@ SimulinkSUL(initScript, paramNames, signalStep, simulinkSimulationStep).use { au
println("cex output: ${verifier.cexOutput[i]}")
}
}
println("Execution time for simulation: ${verifier.simulationTimeSecond} [sec]")
println("Number of simulations: ${verifier.simulinkCount}")
println("Number of simulations for equivalence testing: ${verifier.simulinkCountForEqTest}")
}
105 changes: 105 additions & 0 deletions example/kotlin/ATS6b.main.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
#!/usr/bin/env kscript

// Import the constants for AutoTrans
@file:Import("./AutoTrans.kt")
// This script depends on FalCAuN-core and FalCAuN-matlab
@file:DependsOn("net.maswag:FalCAuN-core:1.0-SNAPSHOT", "net.maswag:FalCAuN-matlab:1.0-SNAPSHOT")
// We assume that the MATLAB_HOME environment variable is set
@file:KotlinOptions("-Djava.library.path=$MATLAB_HOME/bin/maca64/:$MATLAB_HOME/bin/maci64:$MATLAB_HOME/bin/glnxa64")

import net.maswag.*
import kotlin.streams.toList
import java.io.BufferedReader
import java.io.StringReader

// Define the input and output mappers
val throttleValues = listOf(0.0, 50.0, 100.0)
val brakeValues = listOf(0.0, 325.0)
val inputMapper = InputMapperReader.make(listOf(throttleValues, brakeValues))
val velocityValues = listOf(50.0, null)
val accelerationValues = listOf(3000.0, null)
val ignoreValues = listOf(null)
val gearValues = listOf(null)
//val outputMapperReader = OutputMapperReader(listOf(velocityValues, accelerationValues, gearValues, velocityValues, accelerationValues))
val outputMapperReader = OutputMapperReader(listOf(ignoreValues, ignoreValues, gearValues, velocityValues, accelerationValues))
outputMapperReader.parse()
val mapperString = listOf("previous_max_output(0)", "previous_max_output(1)").joinToString("\n")
val signalMapper: ExtendedSignalMapper = ExtendedSignalMapper.parse(BufferedReader(StringReader(mapperString)))
val mapper =
NumericSULMapper(inputMapper, outputMapperReader.largest, outputMapperReader.outputMapper, signalMapper)

// Define the output signal names
val velocity = "signal(0)"
val rotation = "signal(1)"
val gear = "signal(2)"
// Pseudo signals representing the maximum and minimum values between sampling points
// These signals exclude the begin time and include the end time
val prevMaxVelocity = "output(3)"
val prevMaxRotation = "output(4)"

// Define the STL properties
val stlFactory = STLFactory()
signalStep = 1.0
//val stlGRotationLt3000 = "$rotation < 3000.0 && []_[0, ${(30 / signalStep).toInt()}] ($prevMaxRotation < 3000.0)"
val stlGRotationLt3000 = "[]_[0, ${(30 / signalStep).toInt()}] ($prevMaxRotation < 3000.0)"
val stlNotGRotationLt3000 = "<>_[0, ${(30 / signalStep).toInt()}] ($prevMaxRotation > 3000.0)"
//val STLGVelocityLt50 = "$velocity < 50.0 && []_[0,${(8 / signalStep).toInt()}] ($prevMaxVelocity < 50.0)"
val STLGVelocityLt50 = "[]_[0,${(8 / signalStep).toInt()}] ($prevMaxVelocity < 50.0)"
val stlList = listOf(
//"($stlGRotationLt3000) -> ($STLGVelocityLt35)",
// We use || instead of -> because specification strengthening does not support -> yet
// "(!($stlGRotationLt3000)) || ($STLGVelocityLt35)",
// Similarly, we use <>! instead of ![]
"($stlNotGRotationLt3000) || ($STLGVelocityLt50)",
).stream().map { stlString ->
stlFactory.parse(
stlString,
inputMapper,
outputMapperReader.outputMapper,
outputMapperReader.largest
)
}.toList()
val signalLength = 30
val properties = AdaptiveSTLList(stlList, signalLength)

// Constants for the GA-based equivalence testing
val maxTest = 500000
val populationSize = 200
val crossoverProb = 0.5
val mutationProb = 0.01

// Load the automatic transmission model. This automatically closes MATLAB
SimulinkSUL(initScript, paramNames, signalStep, simulinkSimulationStep).use { autoTransSUL ->
// Configure and run the verifier
val verifier = NumericSULVerifier(autoTransSUL, signalStep, properties, mapper)
// Timeout must be set before adding equivalence testing
verifier.setTimeout(50 * 60) // 50 minutes
// We first try equivalence testing based on corner case inputs
verifier.addCornerCaseEQOracle(signalLength, signalLength / 2)
// Then, we use genetic algorithm
verifier.addGAEQOracleAll(
signalLength,
maxTest,
ArgParser.GASelectionKind.Tournament,
populationSize,
crossoverProb,
mutationProb
)
val result = verifier.run()

// Print the result
if (result) {
println("The property is likely satisfied")
} else {
println("The property is falsified")
for (i in 0 until verifier.cexProperty.size) {
println("${verifier.cexProperty[i]} is falsified by the following counterexample)")
println("cex concrete input: ${verifier.cexConcreteInput[i]}")
println("cex abstract input: ${verifier.cexAbstractInput[i]}")
println("cex output: ${verifier.cexOutput[i]}")
}
}
println("Execution time for simulation: ${verifier.simulationTimeSecond} [sec]")
println("Number of simulations: ${verifier.simulinkCount}")
println("Number of simulations for equivalence testing: ${verifier.simulinkCountForEqTest}")
}
47 changes: 27 additions & 20 deletions example/kotlin/pacemaker.main.kts
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,23 @@ import net.maswag.*
import java.io.BufferedReader
import java.io.StringReader

// Set up the pacemaker model
val initScript = """
%% Init Script for the pacemaker model
%% Load the pacemaker model
mdl = 'Model1_Scenario1_Correct';
load_system(mdl);
% The initial pacing conditions in the Simulink model are used.
init_cond = [];
"""
val paramNames = listOf("LRI")
val signalStep = 0.5
val simulinkSimulationStep = 0.0025

// Define the input and output mappers
val lriValues = listOf(50.0, 90.0)
val lriValues = listOf(50.0, 60.0, 70.0, 80.0, 90.0)
val inputMapper = InputMapperReader.make(listOf(lriValues))
val ignoreValue = listOf(null)
val paceCountValues = listOf(7.999999, 15.000001, null)
Expand All @@ -61,35 +76,21 @@ val prevMinPaceCount = "output(4)" // We do not use the minimum values show as a
// Define the STL properties
val stlFactory = STLFactory()
// Signal must be long enough
val stlSignalLength = "alw_[5,5] $LRL > 0"
val stlGPaceCountLt15 = "($paceCount < 15.000001 && alw_[0,5] $prevMaxPaceCount < 15.000001)"
val stlFPaceCountGt8 = "($paceCount > 7.999999 || ev_[0,5] $prevMaxPaceCount > 7.999999)"
val stlSignalLength = "alw_[${(10 / signalStep).toInt()},${(10 / signalStep).toInt()}] $LRL > 0"
val stlGPaceCountLt15 = "($paceCount < 15.000001 && alw_[0,${(10 / signalStep).toInt()}] $prevMaxPaceCount < 15.000001)"
val stlFPaceCountGt8 = "($paceCount > 7.999999 || ev_[0,${(10 / signalStep).toInt()}] $prevMaxPaceCount > 7.999999)"
val stlList = listOf(
stlFactory.parse(
"($stlSignalLength) -> ($stlGPaceCountLt15 && $stlFPaceCountGt8)",
"(!($stlSignalLength)) || ($stlGPaceCountLt15 && $stlFPaceCountGt8)",
inputMapper,
outputMapperReader.outputMapper,
outputMapperReader.largest
)
)
println(stlList.get(0).toAbstractString())
val signalLength = 6
val signalLength = (12 / signalStep).toInt()
val properties = AdaptiveSTLList(stlList, signalLength)

val initScript = """
%% Init Script for the pacemaker model
%% Load the pacemaker model
mdl = 'Model1_Scenario1_Correct'
load_system(mdl);
% The initial pacing conditions in the Simulink model are used.
init_cond = [];
"""
val paramNames = listOf("LRI")
val signalStep = 2.0
val simulinkSimulationStep = 0.0025

// Constants for the GA-based equivalence testing
val maxTest = 10000
val populationSize = 50
Expand All @@ -102,6 +103,9 @@ SimulinkSUL(initScript, paramNames, signalStep, simulinkSimulationStep).use { pa
val verifier = NumericSULVerifier(pacemakerSUL, signalStep, properties, mapper)
// Timeout must be set before adding equivalence testing
verifier.setTimeout(10 * 60) // 10 minutes
// We first try equivalence testing based on corner case inputs
verifier.addCornerCaseEQOracle(signalLength, signalLength / 2)
// Then, we use genetic algorithm
verifier.addGAEQOracleAll(
signalLength,
maxTest,
Expand All @@ -124,4 +128,7 @@ SimulinkSUL(initScript, paramNames, signalStep, simulinkSimulationStep).use { pa
println("cex output: ${verifier.cexOutput[i]}")
}
}
println("Execution time for simulation: ${verifier.simulationTimeSecond} [sec]")
println("Number of simulations: ${verifier.simulinkCount}")
println("Number of simulations for equivalence testing: ${verifier.simulinkCountForEqTest}")
}

0 comments on commit c819f53

Please sign in to comment.