Commit 5f7c17de authored by Gijs van Cuyck's avatar Gijs van Cuyck

finalized lots of things.

added more command line options.
created analysis code for analyzing the results and stats gathered by
the main algorithm.
parent 8de92c03
......@@ -4,6 +4,8 @@ project(complete_ads)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_FLAGS "-static-libgcc -static-libstdc++ -static")
set(CMAKE_EXE_LINKER_FLAGS "-static-libgcc -static-libstdc++ -static")
MATH(EXPR stack_size "512 * 1024 * 1024") # 512 Mb
set(CMAKE_EXE_LINKER_FLAGS "-Wl,--stack,${stack_size}")
set(SOURCE_FILES main.cpp)
#add_subdirectory("lib")
......
@echo off
IF EXIST "do_copy" copy ..\cmake-build-debug\main.exe main.exe
IF EXIST "all_benchmarks_compare_results.txt" del "all_benchmarks_compare_results.txt"
IF EXIST "..\outputs\compare_results\all_benchmarks_compare_results.txt" del "..\outputs\compare_results\all_benchmarks_compare_results.txt"
for /R "..\benchmarks\" %%i in (*) do (
echo running agorithm for %%~nxi
main.exe -c -C "all_benchmarks_compare_results.txt" -a -f %%i
echo running algorithm for %%~nxi
main.exe -c -C "all_benchmarks_compare_results.txt" -a -t -f %%i
echo.
)
pause
\ No newline at end of file
file(GLOB_RECURSE sources "*.cpp")
file(GLOB_RECURSE headers "*.hpp")
set(libs)
add_library(common ${headers} ${sources})
target_link_libraries(common ${libs})
target_include_directories(common PUBLIC ".")
......@@ -8,7 +8,7 @@
#include <vector>
#include <string>
namespace analysys{
namespace analysis{
//calculate the total number of symbols in the test suite where the cost of a reset can be chosen manually.
template<typename T>
......
//
// Created by gijsc on 2-1-2019.
//
#include "results_analysis.hpp"
#include <fstream>
#include <vector>
#include <iostream>
#include <regex>
#include <cassert>
namespace analysis
{
const std::regex integer("(-?[0-9]+)");
//either an integer or an optional integer followed by . followed by a positive integer.
const std::regex real_number("(-?[0-9]*\\.[0-9]+)|(-?[0-9]+)");
//return the first part of the first line of the inputstream that matches the regex.
std::string read_regex(const std::regex &regex, std::ifstream &is)
{
std::smatch m;
std::string line;
std::getline(is, line);
bool found_the_regex = std::regex_search(line, m, regex);
assert(found_the_regex);
return m[0];
}
//returns the first integer on the first line of the input stream
int read_int(std::ifstream &is)
{
return stoi(read_regex(integer, is));
}
//returns the first double on the first line of the inputstream.
double read_double(std::ifstream &is)
{
return stod(read_regex(real_number, is));
}
struct result
{
//amount of states in the input graph
int states;
//amount of different inputs in the input graph
int inputs;
//amount of states the ads method reduce a state identification query to on average.
int average_uncertainty;
//amount of states the ads method reduce a state identification query to in the worst case.
int max_uncertainty;
//percentage difference between the amount of states and the average uncertainty.
double average_uncertainty_gain;
//percentage difference between the amount of states and the max uncertainty.
double max_uncertainty_gain;
//the 'length' of the test suite as generated by the complete ADS method
int complete_ts_length;
//the amount of resets in test suite as generated by the complete ADS method
int complete_ts_resets;
//the 'length' of the test suite as generated by the hybrid ADS method
int hybrid_ts_length;
//the amount of resets in test suite as generated by the hybrid ADS method
int hybrid_ts_resets;
//percentage difference between the complete and hybrid ts length
double length_gain;
//percentage difference between the complete and hybrid ts reset amount.
double resets_gain;
};
result read_results_from_stream(std::ifstream &is)
{
is.ignore(5000, '\n');
is.ignore(5000, '\n');
int states = read_int(is);
int inputs = read_int(is);
is.ignore(5000, '\n');
int average_uncertainty = read_int(is);
int max_uncertainty = read_int(is);
double average_uncertainty_gain = read_double(is);
double max_uncertainty_gain = read_double(is);
is.ignore(5000, '\n');
int complete_ts_length = read_int(is);
int complete_ts_resets = read_int(is);
is.ignore(5000, '\n');
int hybrid_ts_length = read_int(is);
int hybrid_ts_resets = read_int(is);
is.ignore(5000, '\n');
double length_gain = read_double(is);
double resets_gain = read_double(is);
return result{states, inputs, average_uncertainty, max_uncertainty, average_uncertainty_gain,
max_uncertainty_gain,
complete_ts_length, complete_ts_resets, hybrid_ts_length, hybrid_ts_resets, length_gain,
resets_gain};
}
void parse_file(const std::string &input_filename, std::vector<result> &results)
{
std::ifstream input_file(input_filename);
while (!input_file.eof())
{
char lookahead = input_file.peek();
if (lookahead == '\n' || input_file.eof())
{
input_file.ignore(1, '\n');
} else if (lookahead == '#')
{
input_file.ignore(5000, '\n');
} else
{
results.emplace_back(read_results_from_stream(input_file));
}
}
}
void analyze_with_filter( std::ostream &output, std::function<bool(result)> filter,const std::vector<result> & results)
{
int total_cases =0;
int ambigu = 0;
int len_impr_5 = 0;
int len_impr_10 = 0;
int len_impr_30 = 0;
int len_decr_5 = 0;
int len_decr_10 = 0;
int len_decr_30 = 0;
int len_decr_50 = 0;
int len_decr_100 = 0;
int res_impr_5 = 0;
int res_impr_10 = 0;
int res_impr_30 = 0;
int res_decr_5 = 0;
int res_decr_10 = 0;
int res_decr_30 = 0;
int res_decr_50 = 0;
int res_decr_100 = 0;
for (const result & res:results)
{
if(!filter(res))
continue;
total_cases+=1;
if((res.length_gain>0 && res.resets_gain <0)||(res.length_gain<0 && res.resets_gain>0))
ambigu+=1;
if(res.length_gain>=30)
len_impr_30+=1;
if(res.length_gain>=10)
len_impr_10+=1;
if(res.length_gain>=5)
len_impr_5+=1;
if(res.length_gain<=-5)
len_decr_5+=1;
if(res.length_gain<=-10)
len_decr_10+=1;
if(res.length_gain<=-30)
len_decr_30+=1;
if(res.length_gain<=-50)
len_decr_50+=1;
if(res.length_gain<=-100)
len_decr_100+=1;
if(res.resets_gain>=30)
res_impr_30+=1;
if(res.resets_gain>=10)
res_impr_10+=1;
if(res.resets_gain>=5)
res_impr_5+=1;
if(res.resets_gain<=-5)
res_decr_5+=1;
if(res.resets_gain<=-10)
res_decr_10+=1;
if(res.resets_gain<=-30)
res_decr_30+=1;
if(res.resets_gain<=-50)
res_decr_50+=1;
if(res.resets_gain<=-100)
res_decr_100+=1;
}
output << "cases that passed the filter: " << total_cases << "\n";
output << "cases where length and reset changes did not match up: " << ambigu <<"\n";
output << "cases where length improved by at least 5%: " << len_impr_5 <<"\n";
output << "cases where length improved by at least 10%: " << len_impr_10 <<"\n";
output << "cases where length improved by at least 30%: " << len_impr_30 <<"\n";
output << "cases where length worsened by at least 5%: " << len_decr_5<<"\n";
output << "cases where length worsened by at least 10%: " << len_decr_10 <<"\n";
output << "cases where length worsened by at least 30%: " << len_decr_30 <<"\n";
output << "cases where length worsened by at least 50%: " << len_decr_50 <<"\n";
output << "cases where length worsened by at least 100%: " << len_decr_100 <<"\n";
output << "cases where the resets improved by at least 5%: " << res_impr_5 <<"\n";
output << "cases where the resets improved by at least 10%: " << res_impr_10 <<"\n";
output << "cases where the resets improved by at least 30%: " << res_impr_30 <<"\n";
output << "cases where the resets worsened by at least 5%: " << res_decr_5 <<"\n";
output << "cases where the resets worsened by at least 10%: " << res_decr_10 <<"\n";
output << "cases where the resets worsened by at least 30%: " << res_decr_30 <<"\n";
output << "cases where the resets worsened by at least 50%: " << res_decr_50 <<"\n";
output << "cases where the resets worsened by at least 100%: " << res_decr_100 <<"\n";
}
void analyze_results(std::string input_filename, std::string output_directory)
{
size_t start = input_filename.find_last_of('/');
if (start == std::string::npos)
start = input_filename.find_last_of('\\');
size_t end = input_filename.find_last_of('.');
std::string output_filename = output_directory+input_filename.substr(start, end - start) + "_analysis_results.txt";
std::vector<result> results;
parse_file(input_filename, results);
std::ofstream output_file(output_filename);
output_file << "no extra filter\n\n";
output_file << "all results"<<"\n\n";
analyze_with_filter(output_file,[](const result & res){ return true;},results);
output_file << "\n\n";
output_file << "50 or more states"<<"\n\n";
analyze_with_filter(output_file,[](const result & res){ return res.states>=50;},results);
output_file << "\n\n";
output_file << "200 or more states"<<"\n\n";
analyze_with_filter(output_file,[](const result & res){ return res.states>=200;},results);
output_file << "\n\n";
output_file << "10 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[](const result & res){ return res.inputs>=10;},results);
output_file << "\n\n";
output_file << "50 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[](const result & res){ return res.inputs>=50;},results);
output_file << "\n\n";
output_file << "70 or more states and 30 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[](const result & res){ return res.states >=70 && res.inputs>=30;},results);
output_file << "\n\n";
output_file<< "just the cases where the ads method is not applicable\n\n";
std::function<bool(result)> ads_applicable = [](const result& res){ return res.max_uncertainty==1;};
output_file << "all results"<<"\n\n";
analyze_with_filter(output_file,[&ads_applicable](const result & res){ return !ads_applicable(res);},results);
output_file << "\n\n";
output_file << "50 or more states"<<"\n\n";
analyze_with_filter(output_file,[&ads_applicable](const result & res){ return res.states>=50 && !ads_applicable(res);},results);
output_file << "\n\n";
output_file << "200 or more states"<<"\n\n";
analyze_with_filter(output_file,[&ads_applicable](const result & res){ return res.states>=200&& !ads_applicable(res);},results);
output_file << "\n\n";
output_file << "10 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[&ads_applicable](const result & res){ return res.inputs>=10&& !ads_applicable(res);},results);
output_file << "\n\n";
output_file << "50 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[&ads_applicable](const result & res){ return res.inputs>=50&& !ads_applicable(res);},results);
output_file << "\n\n";
output_file << "70 or more states and 30 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[&ads_applicable](const result & res){ return res.states >=70 && res.inputs>=30&& !ads_applicable(res);},results);
output_file << "\n\n";
output_file<< "just the cases where the ads method gives almost no info\n\n";
std::function<bool(result)> ads_somewhat_applicable = [](const result& res){ return res.max_uncertainty<=res.states/2;};
output_file << "all results"<<"\n\n";
analyze_with_filter(output_file,[&ads_somewhat_applicable](const result & res){ return !ads_somewhat_applicable(res);},results);
output_file << "\n\n";
output_file << "50 or more states"<<"\n\n";
analyze_with_filter(output_file,[&ads_somewhat_applicable](const result & res){ return res.states>=50 && !ads_somewhat_applicable(res);},results);
output_file << "\n\n";
output_file << "200 or more states"<<"\n\n";
analyze_with_filter(output_file,[&ads_somewhat_applicable](const result & res){ return res.states>=200&& !ads_somewhat_applicable(res);},results);
output_file << "\n\n";
output_file << "10 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[&ads_somewhat_applicable](const result & res){ return res.inputs>=10&& !ads_somewhat_applicable(res);},results);
output_file << "\n\n";
output_file << "50 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[&ads_somewhat_applicable](const result & res){ return res.inputs>=50&& !ads_somewhat_applicable(res);},results);
output_file << "\n\n";
output_file << "70 or more states and 30 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[&ads_somewhat_applicable](const result & res){ return res.states >=70 && res.inputs>=30&& !ads_somewhat_applicable(res);},results);
output_file << "\n\n";
output_file<< "just the cases where the ads method gives no info\n\n";
std::function<bool(result)> ads_no_extra_info = [](const result& res){ return res.max_uncertainty==res.states;};
output_file << "all results"<<"\n\n";
analyze_with_filter(output_file,[&ads_no_extra_info](const result & res){ return ads_no_extra_info(res);},results);
output_file << "\n\n";
output_file << "50 or more states"<<"\n\n";
analyze_with_filter(output_file,[&ads_no_extra_info](const result & res){ return res.states>=50 && ads_no_extra_info(res);},results);
output_file << "\n\n";
output_file << "200 or more states"<<"\n\n";
analyze_with_filter(output_file,[&ads_no_extra_info](const result & res){ return res.states>=200&& ads_no_extra_info(res);},results);
output_file << "\n\n";
output_file << "10 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[&ads_no_extra_info](const result & res){ return res.inputs>=10&& ads_no_extra_info(res);},results);
output_file << "\n\n";
output_file << "50 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[&ads_no_extra_info](const result & res){ return res.inputs>=50&& ads_no_extra_info(res);},results);
output_file << "\n\n";
output_file << "70 or more states and 30 or more inputs"<<"\n\n";
analyze_with_filter(output_file,[&ads_no_extra_info](const result & res){ return res.states >=70 && res.inputs>=30&& ads_no_extra_info(res);},results);
output_file << "\n\n";
}
}
//
// Created by gijsc on 2-1-2019.
//
#ifndef COMPLETE_ADS_RESULTS_ANALYSIS_H
#define COMPLETE_ADS_RESULTS_ANALYSIS_H
#include <string>
namespace analysis{
void analyze_results(std::string input_filename,std::string output_directory);
}
#endif //COMPLETE_ADS_RESULTS_ANALYSIS_H
//
// Created by Gijs van Cuyck on 19/12/2018.
//
#ifndef COMPLETE_ADS_SPLIT_TREE_ANALYSYS_HPP
#define COMPLETE_ADS_SPLIT_TREE_ANALYSYS_HPP
#include "../hybrid_lib/splitting_tree.hpp"
#include <algorithm>
namespace analysis
{
int count_leaves(const hybrid::splitting_tree & tree)
{
if(tree.children.empty())
return 1;
else
{
int counter = 0;
for(const hybrid::splitting_tree & child: tree.children)
{
counter += count_leaves(child);
}
return counter;
}
}
double average_uncertainty(const hybrid::splitting_tree & tree)
{
return static_cast<double> (tree.states.size())/count_leaves(tree);
}
int maximum_uncertainty(const hybrid::splitting_tree & tree)
{
if(tree.children.empty())
{
return tree.states.size();
} else
{
int max = 0;
for(const hybrid::splitting_tree & child: tree.children)
{
max = std::max(max,maximum_uncertainty(child));
}
return max;
}
}
}
#endif //COMPLETE_ADS_SPLIT_TREE_ANALYSYS_HPP
......@@ -19,8 +19,10 @@
#include "../hybrid_lib/trie.hpp"
#include "../hybrid_lib/test_suite.hpp"
//analysys code
#include "../lib/TS_analysys.hpp"
//analysis code
#include "../lib/TS_analysis.hpp"
#include "../lib/splitting_tree_analysis.hpp"
#include "../lib/results_analysis.hpp"
//stdlib code
#include <algorithm>
......@@ -52,10 +54,12 @@ static const char USAGE[] =
Options:
-h Show this screen
-v Show version
-t print the test suites in the output file. (grow very fast with large input)
-j only run the input through the hybrid-ads method. used mainly for testing
-d overwrite old comparison results file contents instead of appending.
-c compare the resulting test suite with the results given by the hybrid ads code.
-a treat the input file name as an absolute path instead of a relative one.
-r <filename> analyze the compare result file and print some statistics.
-C <filename> the file in which to store compare results. dont sepcify for default filename.
The path should be relative to the outputs/compare_results directory.
This value is only used if the -c options is given, otherwise it is ignored.
......@@ -79,9 +83,11 @@ const std::string compare_directory = output_directory + "compare_results/";
struct main_options
{
bool help = false;
bool version = false;
bool do_compare = false;
bool reset_compare_file = false;
bool print_test_suites = false;
bool only_hybrid = false;
bool do_compare_file_analysis = false;
//"coffe_machine.dot";
//"lee_yannakakis_difficult.dot";
......@@ -89,14 +95,17 @@ struct main_options
//"esm-manual-controller.dot";
//"ABP_Sender.flat_0_1.dot";
//"ex5_with_loops_with_hidden_states_minimized.dot";
std::string input_filename = input_directory + "no_semi-valid_transitions.dot";
//ram_test_minimized.dot
//"../benchmarks/principle/BenchmarkCircuits/ram_test_minimized.dot";
std::string input_filename =
input_directory + "train4_with_loops_with_hidden_states_minimized.dot";
std::string output_filename = "";
std::string hybrid_output_filename = "";
std::string compare_filename = compare_directory + "compare_results.txt";
//extra options for the hybrid code
unsigned long k_max = 1; // 3 means 2 extra states
//todo: figure out what this does
//todo: ask joshua what this does
unsigned long l = 0; // length 0, 1 will be redundancy free
unsigned long rnd_length = 0; // in addition to k_max
unsigned long seed = 0; // 0 for unset/noise
......@@ -110,15 +119,18 @@ main_options parse_options(int argc, char **argv)
try
{
int c;
while ((c = complete::getopt(argc, argv, "hvdcaC:k:x:f:O:o:")) != -1)
while ((c = complete::getopt(argc, argv, "htjdcar:C:k:x:f:O:o:")) != -1)
{
switch (c)
{
case 'h': // show help message
opts.help = true;
break;
case 'v': // show version
opts.version = true;
case 't': //print test suites
opts.print_test_suites = true;
break;
case 'j': //run just the hybrid code
opts.only_hybrid = true;
break;
case 'd': //overwrite old compare file contents.
opts.reset_compare_file = true;
......@@ -129,6 +141,10 @@ main_options parse_options(int argc, char **argv)
case 'a': //use absolute input paths
input_directory = "";
break;
case 'r': //analyze compare results file
opts.do_compare_file_analysis = true;
opts.compare_filename = compare_directory + complete::optarg;
break;
case 'C': //change compare results file
opts.compare_filename = compare_directory + complete::optarg;
break;
......@@ -190,7 +206,7 @@ int main(int argc, char *argv[])
{
/*
* First we parse the command line options.
* We quit when asked for help or version
* We quit when asked for help.
*/
const auto args = parse_options(argc, argv);
......@@ -200,9 +216,9 @@ int main(int argc, char *argv[])
exit(0);
}
if (args.version)
if (args.do_compare_file_analysis)
{
std::cout << "Version 1.4 (December 2018)" << std::endl;
analysis::analyze_results(args.compare_filename, compare_directory);
exit(0);
}
......@@ -245,47 +261,67 @@ int main(int argc, char *argv[])
std::vector<std::string> state_translation = complete::create_reverse_map(translation.state_indices);
//only useful for debugging
/*
ofstream translation_file(output_directory + "translations.txt");
translation_file << "input translation\n";
translation_file << input_translation;
translation_file << "\noutput translation\n";
translation_file << create_reverse_map(translation.output_indices);
translation_file << "\nstate translation\n";
translation_file << state_translation;
translation_file.close();
*/
bool print_translation_help = true;
if (print_translation_help)
{
using namespace complete;
std::ofstream translation_file(output_directory + "translations.txt");
translation_file << "input translation\n";
translation_file << input_translation;
translation_file << "\noutput translation\n";
translation_file << create_reverse_map(translation.output_indices);
translation_file << "\nstate translation\n";
translation_file << state_translation;
translation_file.close();
}
int complete_ts_lenght;
int complete_ts_resets;
if (args.do_compare || !args.only_hybrid)
{
std::cout << "generating complete style splitting tree\n";
complete::splitting_tree complete_splitting_tree = create_splitting_tree(machine);
complete::readable_splitting_tree translated_tree = translate_splitting_tree(complete_splitting_tree,
input_translation,
state_translation);
std::ofstream out_file(args.output_filename);
out_file << "the splitting tree:\n\n";
rst_to_stream(translated_tree, out_file) << "\n\n\n";
std::cout << "generating complete style splitting tree\n";
complete::splitting_tree complete_splitting_tree = create_splitting_tree(machine);
std::cout << "generating complete style separating family\n";
complete::separating_family family = create_separating_family(complete_splitting_tree, machine);
complete::readable_splitting_tree translated_tree = translate_splitting_tree(complete_splitting_tree,
input_translation,
state_translation);
std::ofstream out_file(args.output_filename);
out_file << "the splitting tree:\n\n";
rst_to_stream(translated_tree, out_file) << "\n\n\n";
//making sure the results are actually correct.
assert(test_separating_family(family, machine));
std::cout << "generating complete style separating family\n";
complete::separating_family family = create_separating_family(complete_splitting_tree, machine);
//making sure the results are actually correct.
assert(test_separating_family(family, machine));
out_file << "the separating family:\n\n";
complete::separating_family_to_stream(out_file, family, input_translation, state_translation);
out_file << "the separating family:\n\n";
complete::separating_family_to_stream(out_file, family, input_translation, state_translation);
std::cout << "generating Wp style test suite\n";
complete::test_suite TS = create_test_suite(family, machine, complete::state(0));
std::cout << "generating Wp style test suite\n";
complete::test_suite TS = create_test_suite(family, machine, complete::state(0));
if (args.print_test_suites)
{
out_file << "the test suite:\n\n";
complete::test_suite_to_stream(out_file, TS, input_translation);
}
if (args.do_compare)
{
complete_ts_lenght = analysis::calculate_TS_lenght(TS);
complete_ts_resets = TS.size();
}
out_file.close();
}
out_file << "the test suite:\n\n";
complete::test_suite_to_stream(out_file, TS, input_translation);
out_file.close();
if (args.do_compare)
if (args.do_compare || args.only_hybrid)
{
//create a test suite using the hybrid ads code as well to compare with.
//starts from scratch to avoid the previous calculations from interfering.
......@@ -344,13 +380,16 @@ int main(int argc, char *argv[])
}();
hybrid::splitting_tree hybrid_splitting_tree(0, 0);
auto sequence = [&]
{
const auto tree = [&]
{
std::cout << "generating hybrid style splitting tree\n";
return create_splitting_tree(machine, hybrid::lee_yannakakis_style, random_seeds[1]);
auto result = create_splitting_tree(machine, hybrid::lee_yannakakis_style, random_seeds[1]);
hybrid_splitting_tree = result.root;
return result;
}();