trianta/bayesian-spam-filter
1
0
Fork 0
generated from trianta/cpp-unity-template
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: trianta:cb83e074dd70317e9090fe043829bfbfd1259ee1
Branches Tags
trianta:master
trianta:dev
..
compare: trianta:9b3d373ee02e4935068a3404750e4eb58bb185e9
Branches Tags
trianta:master
trianta:dev

No commits in common. "cb83e074dd70317e9090fe043829bfbfd1259ee1" and "9b3d373ee02e4935068a3404750e4eb58bb185e9" have entirely different histories.
cb83e074dd ... 9b3d373ee0

3 changed files with 38 additions and 107 deletions
Show Stats Expand all files Collapse all files

117
src/filter.cpp
View File

@ -13,12 +13,11 @@ SMSMessage::SMSMessage(bool given_type, std::string given_message) {
}
// Takes in a file name from which it will generate a probability filter
void SMSMessageFilter::GenerateProbability() {
void SMSMessageFilter::GenerateProbability(std::string file_name) {
// Open file
std::ifstream input_messages(generation_file_path);
std::ifstream input_messages(file_name);
if (!input_messages.is_open()) {
std::cerr << "[SMSMessageFilter - Error - Generator] ";
std::cerr << "Error opening '" << generation_file_path << "'" << std::endl;
std::cerr << "Error opening '" << file_name << "'" << std::endl;
std::exit(EXIT_FAILURE);
}
@ -32,16 +31,10 @@ void SMSMessageFilter::GenerateProbability() {
std::getline(full_message, token, '\t');
if (token == "ham") { is_ham_temp = true; }
else if (token == "spam") { is_ham_temp = false; }
else if (token.empty()) {
temp_message.clear();
full_message.clear();
continue;
}
else {
std::cerr << "[SMSMessageFilter - Warning]"
<< " Could not determine message type (probably bad cut)"
<< " ignoring..." << std::endl;
std::cerr << "\t Contains: '" << token << "'" << std::endl;
std::cerr << "[SMSMessageFilter - Warning]" <<
" Could not determine message type" << std::endl;
std::cerr << "\t Contains: <" << token << ">" << std::endl;
temp_message.clear();
full_message.clear();
continue; // Probably a bad line cut
@ -49,16 +42,13 @@ void SMSMessageFilter::GenerateProbability() {
while (std::getline(full_message, token, ' ')) {
token = SanitizeToken(token);
if (token.empty()) { continue; }
if (probability_dictionary[token] == 0) {
probability_dictionary[token] = 0.5;
}
if (is_ham_temp) {
probability_dictionary[token].value += probability_dictionary[token].value * 0.99;
if (probability_dictionary[token].value > 1.) {
probability_dictionary[token].value = 1.;
}
probability_dictionary[token] += probability_dictionary[token] * 0.1;
} else {
probability_dictionary[token].value -= probability_dictionary[token].value * 0.000000001;
if (probability_dictionary[token].value <= 0.) {
probability_dictionary[token].value = 0.0000000001;
}
probability_dictionary[token] -= probability_dictionary[token] * 0.1;
}
}
temp_message.clear();
@ -66,12 +56,11 @@ void SMSMessageFilter::GenerateProbability() {
}
}
void SMSMessageFilter::Prepare() {
void SMSMessageFilter::Prepare(std::string file_name) {
// Open file
std::ifstream input_messages(filter_file_path);
std::ifstream input_messages(file_name);
if (!input_messages.is_open()) {
std::cerr << "[SMSMessageFilter - Error - Filter] ";
std::cerr << "Error opening '" << filter_file_path << "'" << std::endl;
std::cerr << "Error opening '" << file_name << "'" << std::endl;
std::exit(EXIT_FAILURE);
}
@ -85,20 +74,15 @@ void SMSMessageFilter::Prepare() {
std::getline(full_message, token, '\t');
if (token == "ham") { is_ham_temp = true; }
else if (token == "spam") { is_ham_temp = false; }
else if (token.empty()) {
temp_message.clear();
full_message.clear();
continue;
}
else {
std::cerr << "[SMSMessageFilter - Warning]"
<< " Could not determine message type (probably bad cut)"
<< " ignoring..." << std::endl;
std::cerr << "\t Contains: '" << token << "'" << std::endl;
std::cerr << "[SMSMessageFilter - Warning]" <<
" Could not determine message type" << std::endl;
std::cerr << "\t Contains: <" << token << ">" << std::endl;
temp_message.clear();
full_message.clear();
continue; // Probably a bad line cut
}
full_message.ignore('\t');
filtered_messages.emplace_back(is_ham_temp, full_message.str());
temp_message.clear();
full_message.clear();
@ -113,16 +97,15 @@ void SMSMessageFilter::Filter(void) {
full_message.str(filtered_messages[i].message);
while (std::getline(full_message, token, ' ')) {
token = SanitizeToken(token);
type_probability = probability_dictionary[token].value * type_probability;
if (probability_dictionary[token] == 0) {
probability_dictionary[token] = 0.5;
}
type_probability = probability_dictionary[token] * type_probability;
}
double final_probability;
final_probability = (1. - sentence_probability_ham) * type_probability;
final_probability = final_probability / (final_probability + ((1. - type_probability) * sentence_probability_ham));
if (final_probability >= sentence_probability_ham) {
if (type_probability <= sentence_probability_ham) {
filtered_messages[i].is_ham_filter = true;
} else { filtered_messages[i].is_ham_filter = false; }
type_probability = 0.5;
full_message.clear();
}
}
@ -130,36 +113,11 @@ void SMSMessageFilter::Report(void) {
PrintReport(GenerateReport());
}
void SMSMessageFilter::ReadArguments(int argc, char* argv[]) {
if (argc == 1) {
std::cerr << "Usage error: Please specify file to filter" << std::endl;
PrintHelp();
std::exit(1);
}
std::string argument_string;
for (int i = 0; i < argc; ++i) {
argument_string.assign(argv[i]);
if (argument_string == "-g") {
is_generator_defined = true;
generation_file_path.assign(argv[i+1]);
}
if (argument_string == "-f") {
is_input_defined = true;
filter_file_path.assign(argv[i+1]);
}
}
if (!is_input_defined) {
std::cerr << "Usage error: Please specify file to filter" << std::endl;
PrintHelp();
std::exit(1);
}
}
ReportData SMSMessageFilter::GenerateReport(void) {
double true_ham = 0.;
double true_spam = 0.;
double false_ham = 0.;
double false_spam = 0.;
double true_ham = 0;
double true_spam = 0;
double false_ham = 0;
double false_spam = 0;
for (SMSMessage message : filtered_messages) {
// Get total count
if (!(message.is_ham ^ message.is_ham_filter)) {
@ -171,7 +129,6 @@ ReportData SMSMessageFilter::GenerateReport(void) {
}
}
std::cout << std::endl;
std::cout << "[SMSMessageFilter - Info] Ham barrier: ";
std::cout << sentence_probability_ham << std::endl;
std::cout << "[SMSMessageFilter - Info] True ham: ";
@ -182,14 +139,13 @@ ReportData SMSMessageFilter::GenerateReport(void) {
std::cout << false_ham << std::endl;
std::cout << "[SMSMessageFilter - Info] False spam: ";
std::cout << false_spam << std::endl;
std::cout << std::endl;
// Calculate report data
ReportData new_report;
new_report.ham_precision = (true_ham) / (true_ham + false_ham);
new_report.ham_recall = (true_ham) / (true_ham + false_spam);
new_report.spam_precision = (true_spam) / (true_spam + false_spam);
new_report.spam_recall = (true_spam) / (true_spam + false_ham);
new_report.spam_precision = (true_ham) / (true_ham + false_ham);
new_report.spam_recall = (true_ham) / (true_ham + false_spam);
new_report.ham_precision = (true_spam) / (true_spam + false_spam);
new_report.ham_recall = (true_spam) / (true_spam + false_ham);
new_report.spam_f_score = 2.0 * (new_report.spam_precision * new_report.spam_recall) / (new_report.spam_precision + new_report.spam_recall);
new_report.ham_f_score = 2.0 * (new_report.ham_precision * new_report.ham_recall) / (new_report.ham_precision + new_report.ham_recall);
new_report.accuracy = (new_report.spam_recall + new_report.ham_recall) / 2.0;
@ -197,8 +153,6 @@ ReportData SMSMessageFilter::GenerateReport(void) {
}
void SMSMessageFilter::PrintReport(ReportData report) {
std::cout << std::endl;
std::cout << "============ [SMSMessageFilter - Report - Start] ============" << std::endl;
// Spam precision: (true positives) / (true positives + false positives)
std::cout << "[SMSMessageFilter - Report] Spam precision: ";
std::cout << report.spam_precision << std::endl;
@ -226,7 +180,6 @@ void SMSMessageFilter::PrintReport(ReportData report) {
// Accuracy: (spam recall + ham recall) / 2
std::cout << "[SMSMessageFilter - Report] Accuracy: ";
std::cout << report.accuracy << std::endl;
std::cout << "============ [SMSMessageFilter - Report - End] ============" << std::endl;
}
std::string SanitizeToken(std::string token) {
@ -239,11 +192,3 @@ std::string SanitizeToken(std::string token) {
[](unsigned char c){ return std::tolower(c); });
return token;
}
void PrintHelp(void) {
std::cerr << "Usage:\tfilter [-g file_for_probability] -f file_to_be_filtered" << std::endl;
std::cerr << "\t -g description: Generates probabilities of words from this file" << std::endl;
std::cerr << "\t\t (If not specified, then all words default to a 0.5 weight)" << std::endl;
std::cerr << "\t -f description: File that will actually be used to filter from" << std::endl;
std::cerr << "\t\t (Report is generated from this)" << std::endl;
}

19
src/filter.hpp
View File

@ -5,11 +5,6 @@
#include <string>
#include <vector>
// C++ maps don't allow specifying default value
struct DoubleDefaultedToHalf {
double value = 0.5;
};
struct ReportData {
double spam_precision;
double spam_recall;
@ -31,19 +26,14 @@ class SMSMessageFilter {
public:
SMSMessageFilter(void) = default;
~SMSMessageFilter(void) = default;
bool is_generator_defined = false;
bool is_input_defined = false;
void GenerateProbability();
void Prepare();
void GenerateProbability(std::string file_name);
void Prepare(std::string file_name);
void Filter(void);
void Report(void);
void ReadArguments(int argc, char* argv[]);
private:
double sentence_probability_ham = 0.2; // Sentence is spam if < this value
std::map<std::string, DoubleDefaultedToHalf> probability_dictionary;
std::string generation_file_path;
std::string filter_file_path;
double sentence_probability_ham = 0.5; // Spam is 1 - sentence_probability_ham
std::map<std::string, double> probability_dictionary;
std::vector<SMSMessage> filtered_messages;
ReportData GenerateReport(void);
void PrintReport(ReportData report);
@ -51,6 +41,5 @@ private:
};
std::string SanitizeToken(std::string token);
void PrintHelp(void);
#endif // !FILTER_HPP

9
src/main.cpp
View File

@ -1,12 +1,9 @@
#include "filter.hpp"
int main(int argc, char* argv[]) {
int main(void) {
SMSMessageFilter single_filter;
single_filter.ReadArguments(argc, argv);
if (single_filter.is_generator_defined) {
single_filter.GenerateProbability();
}
single_filter.Prepare();
single_filter.GenerateProbability("test/SMSProbabilityGeneration.txt");
single_filter.Prepare("test/SMSFilterTest.txt");
single_filter.Filter();
single_filter.Report();
return 0;