Commit 5c0bb941 authored by Gijs van Cuyck's avatar Gijs van Cuyck

integrated the hybrid ads code into the main method and added command

line option to use it in addition to the regular code and compare the
results.
the results of the comparison get added to a different file that can be
changed through command line options.
parent d4a2b4e2
No preview for this file type
@echo off
IF EXIST "do_copy" copy ..\cmake-build-debug\main.exe main.exe
IF EXIST "all_benchmarks_compare_results.txt" del "all_benchmarks_compare_results.txt"
for /R "..\benchmarks\" %%i in (*) do (
echo running agorithm for %%~nxi
main.exe -a -f %%i
main.exe -c -C "all_benchmarks_compare_results.txt" -a -f %%i
echo.
)
pause
\ No newline at end of file
//
// Created by Gijs van Cuyck on 28/11/2018.
// code related to analysing test suites
//
#ifndef COMPLETE_ADS_TS_ANALYSYS_HPP
#define COMPLETE_ADS_TS_ANALYSYS_HPP
#include <vector>
#include <string>
namespace analysys{
//calculate the total number of symbols in the test suite where the cost of a reset can be chosen manually.
template<typename T>
int calculate_TS_lenght(const std::vector<std::vector<T>> & TS, int reset_cost = 1)
{
int result = 0;
for(const std::vector<T> & test : TS)
{
result+=test.size()+reset_cost;
}
return result;
}
}
#endif //COMPLETE_ADS_TS_ANALYSYS_HPP
......@@ -19,7 +19,10 @@
#include "../hybrid_lib/trie.hpp"
#include "../hybrid_lib/test_suite.hpp"
//analysys code
#include "../lib/TS_analysys.hpp"
//stdlib code
#include <algorithm>
#include <cstdlib>
#include <iostream>
......@@ -41,8 +44,6 @@ extern "C" {
#include <unistd.h>
#endif
using namespace std;
static const char USAGE[] =
R"(Generate or stream a test suite for a given FSM.
......@@ -52,6 +53,7 @@ static const char USAGE[] =
Options:
-h Show this screen
-v Show version
-d overwrite old comparison results file contents instead of appending.
-c compare the resulting test suite with the results given by the hybrid ads code.
-a treat the input file name as an absolute path instead of a relative one.
-C <filename> the file in which to store compare results. dont sepcify for default filename.
......@@ -63,18 +65,20 @@ static const char USAGE[] =
-o <filename> Output filename ('-' or don't specify for default output file). path should be relative to outputs directory
)";
const string output_directory = "../outputs/";
string input_directory = "../examples/";
const string compare_directory = output_directory + "compare_results/";
const std::string output_directory = "../outputs/";
std::string input_directory = "../examples/";
const std::string compare_directory = output_directory + "compare_results/";
//extra datatypes for the hybrid code
struct main_options
{
bool help = false;
bool version = false;
bool do_compare = false;
bool reset_compare_file = false;
//"coffe_machine.dot";
//"lee_yannakakis_difficult.dot";
......@@ -82,10 +86,10 @@ struct main_options
//"esm-manual-controller.dot";
//"ABP_Sender.flat_0_1.dot";
//"ex5_with_loops_with_hidden_states_minimized.dot";
string input_filename = input_directory + "lee_yannakakis_difficult.dot";
string output_filename = "";
string hybrid_output_filename = "";
string compare_filename = compare_directory + "compare_results.txt";
std::string input_filename = input_directory + "lee_yannakakis_difficult.dot";
std::string output_filename = "";
std::string hybrid_output_filename = "";
std::string compare_filename = compare_directory + "compare_results.txt";
//extra options for the hybrid code
unsigned long k_max = 1; // 3 means 2 extra states
......@@ -103,7 +107,7 @@ main_options parse_options(int argc, char **argv)
try
{
int c;
while ((c = complete::getopt(argc, argv, "hvcaC:k:x:f:o:")) != -1)
while ((c = complete::getopt(argc, argv, "hvdcaC:k:x:f:o:")) != -1)
{
switch (c)
{
......@@ -113,6 +117,9 @@ main_options parse_options(int argc, char **argv)
case 'v': // show version
opts.version = true;
break;
case 'd': //overwrite old compare file contents.
opts.reset_compare_file = true;
break;
case 'c': //compare results with hybrid code
opts.do_compare = true;
break;
......@@ -123,10 +130,10 @@ main_options parse_options(int argc, char **argv)
opts.compare_filename = compare_directory + complete::optarg;
break;
case 'k': // select extra states / k-value
opts.k_max = stoul(complete::optarg);
opts.k_max = std::stoul(complete::optarg);
break;
case 'x': // seed
opts.seed = stoul(complete::optarg);
opts.seed = std::stoul(complete::optarg);
break;
case 'f': // input filename
opts.input_filename = input_directory + complete::optarg;
......@@ -136,17 +143,17 @@ main_options parse_options(int argc, char **argv)
opts.hybrid_output_filename = output_directory + "hybrid_" + complete::optarg;
break;
case ':': // some option without argument
throw runtime_error(string("No argument given to option -") + char(complete::optopt));
throw std::runtime_error(std::string("No argument given to option -") + char(complete::optopt));
case '?': // all unrecognised things
default: //should never happen but included for completeness sake.
throw runtime_error(string("Unrecognised option -") + char(complete::optopt));
throw std::runtime_error(std::string("Unrecognised option -") + char(complete::optopt));
}
}
} catch (exception &e)
} catch (std::exception &e)
{
cerr << e.what() << endl;
cerr << "Could not parse command line options." << endl;
cerr << "Please use -h to see the available options." << endl;
std::cerr << e.what() << std::endl;
std::cerr << "Could not parse command line options." << std::endl;
std::cerr << "Please use -h to see the available options." << std::endl;
exit(2);
}
......@@ -183,13 +190,13 @@ int main(int argc, char *argv[])
if (args.help)
{
cout << USAGE << endl;
std::cout << USAGE << std::endl;
exit(0);
}
if (args.version)
{
cout << "Version 1.4 (December 2018)" << endl;
std::cout << "Version 1.4 (December 2018)" << std::endl;
exit(0);
}
......@@ -204,19 +211,19 @@ int main(int argc, char *argv[])
const auto &filename = args.input_filename;
if (filename.empty() || filename == "-")
{
return complete::read_mealy_from_dot(cin);
return complete::read_mealy_from_dot(std::cin);
}
if (filename.find(".txt") != string::npos)
if (filename.find(".txt") != std::string::npos)
{
auto m = complete::read_mealy_from_txt(filename);
auto t = create_translation_for_mealy(m);
return make_pair(move(m), move(t));
} else if (filename.find(".dot") != string::npos)
return std::make_pair(std::move(m), std::move(t));
} else if (filename.find(".dot") != std::string::npos)
{
return complete::read_mealy_from_dot(filename);
}
clog << "warning: unrecognized file format, assuming .dot\n";
std::clog << "warning: unrecognized file format, assuming .dot\n";
return complete::read_mealy_from_dot(filename);
}();
......@@ -228,8 +235,8 @@ int main(int argc, char *argv[])
const auto &translation = machine_and_translation.second;
vector<string> input_translation = complete::create_reverse_map(translation.input_indices);
vector<string> state_translation = complete::create_reverse_map(translation.state_indices);
std::vector<std::string> input_translation = complete::create_reverse_map(translation.input_indices);
std::vector<std::string> state_translation = complete::create_reverse_map(translation.state_indices);
//only useful for debugging
/*
......@@ -244,18 +251,18 @@ int main(int argc, char *argv[])
*/
cout << "creating splitting tree\n";
std::cout << "creating splitting tree\n";
complete::splitting_tree complete_splitting_tree = create_splitting_tree(machine);
cout << "creating readable splitting tree\n";
std::cout << "creating readable splitting tree\n";
complete::readable_splitting_tree translated_tree = translate_splitting_tree(complete_splitting_tree,
input_translation,
state_translation);
ofstream out_file(args.output_filename);
std::ofstream out_file(args.output_filename);
out_file << "the splitting tree:\n\n";
rst_to_stream(translated_tree, out_file) << "\n\n\n";
cout << "creating separating family\n";
std::cout << "creating separating family\n";
complete::separating_family family = create_separating_family(complete_splitting_tree, machine);
//making sure the results are actually correct.
......@@ -272,46 +279,48 @@ int main(int argc, char *argv[])
complete::test_suite_to_stream(out_file, TS, input_translation);
out_file.close();
cout << "finished!\n";
std::cout << "finished!\n";
if (args.do_compare)
{
//create a test suite using the hybrid ads code as well to compare with.
//starts from scratch to avoid the previous calculations from interfering.
//start of original hybrid test suite code
const auto machine_and_translation = [&]
{
const auto &filename = args.input_filename;
if (filename == "" || filename == "-")
{
return hybrid::read_mealy_from_dot(cin);
return hybrid::read_mealy_from_dot(std::cin);
}
if (filename.find(".txt") != string::npos)
if (filename.find(".txt") != std::string::npos)
{
const hybrid::mealy m = hybrid::read_mealy_from_txt(filename);
const auto t = create_translation_for_mealy(m);
return make_pair(move(m), move(t));
} else if (filename.find(".dot") != string::npos)
return std::make_pair(std::move(m), std::move(t));
} else if (filename.find(".dot") != std::string::npos)
{
return hybrid::read_mealy_from_dot(filename);
}
clog << "warning: unrecognized file format, assuming .dot\n";
std::clog << "warning: unrecognized file format, assuming .dot\n";
return hybrid::read_mealy_from_dot(filename);
}();
const auto &machine = hybrid::reachable_submachine(move(machine_and_translation.first), 0);
const auto &machine = hybrid::reachable_submachine(std::move(machine_and_translation.first), 0);
const auto &translation = machine_and_translation.second;
const auto random_seeds = [&]
{
vector<uint_fast32_t> seeds(4);
std::vector<uint_fast32_t> seeds(4);
if (args.seed != 0)
{
seed_seq s{args.seed};
std::seed_seq s{args.seed};
s.generate(seeds.begin(), seeds.end());
} else
{
random_device rd;
std::random_device rd;
generate(seeds.begin(), seeds.end(), ref(rd));
}
return seeds;
......@@ -348,7 +357,7 @@ int main(int argc, char *argv[])
}();
auto transfer_sequences = create_transfer_sequences(hybrid::minimal_transfer_sequences, machine, 0,
random_seeds[2]);
random_seeds[2]);
auto const inputs = hybrid::create_reverse_map(translation.input_indices);
......@@ -356,38 +365,72 @@ int main(int argc, char *argv[])
hybrid::trie<hybrid::input> test_suite;
hybrid::word buffer;
const auto output_word = [&inputs](const auto & w) {
for (const auto & x : w) {
cout << inputs[x] << ' ';
const auto output_word = [&inputs](const auto &w)
{
for (const auto &x : w)
{
std::cout << inputs[x] << ' ';
}
cout << endl;
std::cout << std::endl;
};
vector<hybrid::word> mid_sequences(1);
hybrid::test(machine, transfer_sequences, mid_sequences, separating_family, args.k_max+1,
{[&buffer](auto const &w)
{ buffer.insert(buffer.end(), w.begin(), w.end()); },
[&buffer, &test_suite]()
{
test_suite.insert(buffer);
buffer.clear();
return true;
}});
std::vector<hybrid::word> mid_sequences(1);
hybrid::test(machine, transfer_sequences, mid_sequences, separating_family, args.k_max + 1,
{[&buffer](auto const &w)
{ buffer.insert(buffer.end(), w.begin(), w.end()); },
[&buffer, &test_suite]()
{
test_suite.insert(buffer);
buffer.clear();
return true;
}});
std::vector<hybrid::word> hybrid_test_suite = flatten(test_suite);
vector<hybrid::word> hybrid_test_suite = flatten(test_suite);
//end of original hybrid test suite code.
//end of hybrid test suite code.
std::vector<std::vector<std::string>> translated_hybrid_TS;
translated_hybrid_TS.reserve(hybrid_test_suite.size());
ofstream hybrid_out_file(args.hybrid_output_filename);
hybrid_out_file<< "the test suite:\n\n";
for(const hybrid::word test : hybrid_test_suite)
std::ofstream hybrid_out_file(args.hybrid_output_filename);
hybrid_out_file << "the test suite:\n\n";
for (const hybrid::word &test : hybrid_test_suite)
{
std::vector<std::string> translated_word(test.size());
transform(test.begin(), test.end(), translated_word.begin(), [&inputs](size_t symbol_index)
{ return inputs[symbol_index]; });
translated_hybrid_TS.push_back(translated_word);
hybrid_out_file << complete::join(translated_word.begin(), translated_word.end(), ",", "\n");
}
hybrid_out_file.close();
std::ofstream result_stats_file;
if (args.reset_compare_file)
result_stats_file = std::ofstream(args.compare_filename);
else
result_stats_file = std::ofstream(args.compare_filename, std::ios_base::app);
size_t start = args.input_filename.find_last_of('/');
if (start == std::string::npos)
start = args.input_filename.find_last_of('\\');
std::string filename = args.input_filename.substr(start);
int complete_ts_lenght = analysys::calculate_TS_lenght(TS);
int hybrid_ts_lenght = analysys::calculate_TS_lenght(hybrid_test_suite);
result_stats_file << "results for " << filename << "\n";
result_stats_file << "complete test suite:\n";
result_stats_file << "\ttotal length: " << complete_ts_lenght << "\n";
result_stats_file << "\tresets: " << TS.size() << "\n";
result_stats_file << "hybrid test suite:\n";
result_stats_file << "\ttotal length: " << hybrid_ts_lenght << "\n";
result_stats_file << "\tresets: " << hybrid_test_suite.size() << "\n";
result_stats_file << "complete test suite improvements: (negative represent worse results)\n";
result_stats_file << "\tpercentage lenth improvement: "
<< 100.0 * (hybrid_ts_lenght - complete_ts_lenght) / (hybrid_ts_lenght) << "%\n";
result_stats_file << "\tpercentage resets improvement: "
<< 100.0 * (int) (hybrid_test_suite.size() - TS.size()) / (hybrid_test_suite.size())
<< "%\n\n";
result_stats_file.close();
}
exit(0);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment