Commit 6602b32e authored by Paul Fiterau Brostean's avatar Paul Fiterau Brostean
Browse files

Updated benchmarks.

parent fb6fcb42
......@@ -3,12 +3,30 @@ from typing import Tuple, List, Dict
import collections
from encode.fa import DFAEncoder, MealyEncoder
from encode.iora import IORAEncoder
from encode.ra import RAEncoder
from learn import Learner
from learn.fa import FALearner
from learn.ra import RALearner
from model import Automaton
from model.ra import RegisterMachine
from sut import SUTType, ScalableSUTClass
from sut.login import LoginClass
from test import TestGenerator
from learn.algorithm import learn_mbt, Statistics
from test.rwalk import DFARWalkFromState, MealyRWalkFromState, RARWalkFromState, IORARWalkFromState
class SutDesc(collections.namedtuple("SutDesc", 'sut_class type size')):
def __str__(self):
return str(self.type) + "_" + str(self.sut_class.__class__.__name__).replace("Class", "") + "(" + str(self.size) + ")"
TestDesc = collections.namedtuple("TestDesc", 'max_tests rand_length prop_reset')
class CollectedStats(collections.namedtuple("CollectedStats", "states registers learn_tests "
"learn_inputs total_tests learn_time")):
SutDesc = collections.namedtuple("SutDesc", 'class type size')
class Benchmark:
def __init__(self):
......@@ -31,26 +49,72 @@ class Benchmark:
return self
def _run_benchmark(self, sut_class:ScalableSUTClass, sut_type:SUTType, learner:Learner,
test_gen:type, max_texts:int, timeout:int) -> List[Tuple[SutDesc, Statistics]]:
test_gen:type, test_desc:TestDesc, tout:int) -> List[Tuple[SutDesc, CollectedStats]]:
results = []
size = 1
while True:
sut = sut_class.new_sut(sut_type, size)
tester = test_gen(sut)
(model, statistics) = learn_mbt(learner, tester, max_texts)
# ugly but there you go
tester = test_gen(sut, test_desc.rand_length, test_desc.prop_reset)
(model, statistics) = learn_mbt(learner, tester, test_desc.max_tests)
if model is None:
results.append(SutDesc(sut_class, sut_type, size), statistics)
imp_stats = self._collect_stats(model, statistics)
results.append( (SutDesc(sut_class, sut_type, size), imp_stats))
size += 1
return results
def run_benchmarks(self, max_texts:int, timeout:int) -> List[Tuple[SutDesc, Statistics]]:
def _collect_stats(self, model:Automaton, statistics:Statistics) -> CollectedStats:
states = len(model.states())
registers = len(model.registers()) if isinstance(model, RegisterMachine) else None
learn_tests = statistics.num_learner_tests
learn_inputs = statistics.num_learner_inputs
total_tests = statistics.suite_size
learn_time = sum(statistics.learning_times)
return CollectedStats(states=states, registers=registers, learn_tests=learn_tests, learn_inputs=learn_inputs,
total_tests=total_tests, learn_time=learn_time)
def run_benchmarks(self, test_desc:TestDesc, timeout:int) -> List[Tuple[SutDesc, CollectedStats]]:
results = []
for sut_class, sut_type in self.suts:
(learner, tester) = self.learn_setup[sut_type]
res = self._run_benchmark(sut_class, sut_type, learner, tester, max_texts, timeout)
res = self._run_benchmark(sut_class, sut_type, learner, tester, test_desc, timeout)
return results
def print_results(results : List[Tuple[SutDesc, CollectedStats]]):
if len(results) == 0:
print ("No statistics to report on")
for sut_desc,stats in results:
print(sut_desc, " ", stats)
b = Benchmark()
# adding learning setups for each type of machine
b.add_setup(SUTType.DFA, FALearner(DFAEncoder()), DFARWalkFromState)
b.add_setup(SUTType.Mealy, FALearner(MealyEncoder()), MealyRWalkFromState)
b.add_setup(SUTType.RA, RALearner(RAEncoder()), RARWalkFromState)
b.add_setup(SUTType.IORA, RALearner(IORAEncoder()), IORARWalkFromState)
# add the sut classes we want to benchmark
b.add_sut(LoginClass(), SUTType.RA)
# create a test description
t_desc = TestDesc(max_tests=10000, prop_reset=0.2, rand_length=5)
# give an smt timeout value (in ms)
timeout = 1000
# run the benchmark and collect results
results = b.run_benchmarks(t_desc, timeout)
# print results
from typing import List, Tuple, Union
from typing import cast
import collections
from model import Automaton
from learn import Learner
from test import TestGenerator, Test
......@@ -11,6 +13,7 @@ __all__ = [
class Statistics():
"""We only refe"""
def __init__(self):
......@@ -18,6 +21,7 @@ class Statistics():
self.num_learner_inputs = 0
self.suite_size = 0
self.learning_times = []
self.model_stats = None
def add_learner_test(self, test:Test):
""" updates the stats with relevant information from the added test"""
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment