Add Tmirror class for mirror benchmarking and option [mirrors].max_connections_num_per_mirror

Segget gathers stats on mirrors and uses it to rate them and therefore improve performance.
settings.forgetness option (by default set to 5) allows to adjust how fast segget "forgets"
about bad performance of a mirror.

7 files changed, 152 insertions, 14 deletions

diff --git a/segget/connection.cpp b/segget/connection.cpp
index b6a2394..58b0bfb 100644
--- a/segget/connection.cpp
+++ b/segget/connection.cpp
@@ -7,8 +7,12 @@ class Tconnection{
 	private:
 		ulong bytes_per_last_interval;
 	public:
+		time_t start_time;
 		void *segment;
-		Tconnection():bytes_per_last_interval(0),segment(0){};
+		Tconnection():
+			bytes_per_last_interval(0),
+			start_time(0),
+			segment(0){};
 		void  inc_bytes_per_last_interval(ulong new_bytes_count){bytes_per_last_interval+=new_bytes_count;};
 		ulong get_bytes_per_last_interval(){return bytes_per_last_interval;};
 		void  reset_bytes_per_last_interval(){bytes_per_last_interval=0;};
diff --git a/segget/distfile.cpp b/segget/distfile.cpp
index 9cbaada..e4f17f0 100644
--- a/segget/distfile.cpp
+++ b/segget/distfile.cpp
@@ -14,6 +14,8 @@
 #include <iostream>
 #include <json/json.h>
 #include "segment.cpp"
+#include "mirror.cpp"
+#include "checksum.cpp"
 
 //ulong max_segment_size=1000*500;
 
@@ -140,11 +142,46 @@ Tdistfile::~Tdistfile(){
 
 void Tdistfile::provide_segment(CURLM* cm, uint connection_num, uint seg_num){
 	active_connections_num++;
-	dn_segments[seg_num].prepare_for_connection(cm, connection_num, num, url_list[url_num]);
-	connection_array[connection_num].segment=&dn_segments[seg_num];
-	url_num++;
-	if (url_num >= url_count)
-		url_num=0;
+
+	Tmirror *Pcurr_mirror;
+	Tmirror *Pbest_mirror;
+//=============================================================================
+//
+//
+//  BEWARE first mirror needs to be checked for settings.max_connections_num_per_mirror
+//
+//
+//=============================================================================
+	
+	// assume first mirrror to be the best and compare it with the rest
+	uint best_mirror_num=-1;
+//	Pcurr_mirror=find_mirror(strip_mirror_name(url_list[best_mirror_num]));
+	Pbest_mirror=0;
+	ulong best_mirror_laziness_criterion=-1;
+	double curr_mirror_laziness_criterion;
+
+	for (url_num=0; url_num<url_count; url_num++){
+		Pcurr_mirror=find_mirror(strip_mirror_name(url_list[url_num]));
+		if (Pcurr_mirror->get_active_num()<settings.max_connections_num_per_mirror){
+			curr_mirror_laziness_criterion=Pcurr_mirror->get_laziness_criterion();
+			if (curr_mirror_laziness_criterion<best_mirror_laziness_criterion){
+				best_mirror_num=url_num;
+				best_mirror_laziness_criterion=curr_mirror_laziness_criterion;
+				Pbest_mirror=Pcurr_mirror;
+			}
+			if (best_mirror_laziness_criterion==0)
+				// 0 can not be improved - it's one of the best
+				break;
+		}
+	}
+	debug("Downloading from BEST_MIRROR:"+url_list[best_mirror_num]);
+	if (Pbest_mirror){
+		Pbest_mirror->start();
+		dn_segments[seg_num].prepare_for_connection(cm, connection_num, num, url_list[best_mirror_num]);
+		connection_array[connection_num].segment=&dn_segments[seg_num];
+	}
+	else
+		error_log("Can't choose mirror for segment:"+dn_segments[seg_num].file_name);
 }
 void Tdistfile::inc_dld_segments_count(Tsegment* current_segment){
 	stats.inc_dld_size(current_segment->segment_size);
@@ -189,6 +226,7 @@ void Tdistfile::combine_segments(){
 	distfile_file.close();
 	stats.inc_dld_distfiles_count();
 	log("Distfile "+name+" has been combined");
+
 	if (rmd160_ok(settings.distfiles_dir+"/"+name,RMD160))
 		log("RMD160 checksum for distfile:"+name+" is [OK]");
 	else{
@@ -209,5 +247,6 @@ void Tdistfile::combine_segments(){
 		log("Error: SHA256 checksum for distfile:"+name+" [FAILED]");
 		error_log("Error: SHA256 checksum for distfile:"+name+" [FAILED]");
 	}
+
 }
 #endif
\ No newline at end of file
diff --git a/segget/mirror.cpp b/segget/mirror.cpp
new file mode 100644
index 0000000..4603baa
--- /dev/null
+++ b/segget/mirror.cpp
@@ -0,0 +1,80 @@
+#ifndef __MIRROR_BENCHMARKER_H__
+#define __MIRROR_BENCHMARKER_H__
+
+#include <string>
+#include <map>
+#include "str.cpp"
+#include "tui.cpp"
+#include "settings.cpp"
+#include "tui.cpp"
+using namespace std;
+
+#define FORGETNESS 20;
+
+class Tmirror{
+	private:
+		uint active_num;
+	public:
+		ulong	dld_time;
+		ulong	dld_size;
+		double	multiplier;
+		Tmirror():
+			active_num(0),
+			dld_time(0),
+			dld_size(1),
+			multiplier(1){};
+		void	start();
+		void	stop(ulong time, uint size);
+		double get_laziness_criterion();
+		uint get_active_num(){return active_num;};
+};
+
+double Tmirror::get_laziness_criterion(){
+	double criterion=multiplier*1000000000*dld_time/dld_size;
+	multiplier=multiplier*100/(100+settings.forgetness);
+	return criterion;
+}
+void Tmirror::start(){
+	if (!dld_time)
+		dld_time=1;
+	active_num++;
+	multiplier=1;
+}
+
+void Tmirror::stop(ulong time, uint size){
+	dld_time+=time;
+	dld_size+=size;
+	multiplier=1;
+	debug(toString(time)+"---"+toString(size));
+	active_num--;
+}
+
+map<string,Tmirror *> mirror_list;
+
+Tmirror* find_mirror(string mirror_url){
+	map<string,Tmirror*>::const_iterator mirror_iterator = mirror_list.find(mirror_url);
+	if (mirror_iterator==mirror_list.end()){
+		Tmirror * Pnew_mirror=new Tmirror;
+		debug("Cant find mirror:"+mirror_url+" - creating new record");
+		mirror_list[mirror_url]=Pnew_mirror;
+		return Pnew_mirror;
+	}
+	else{
+		debug("Found mirror:"+mirror_url);
+		debug("==================>>");
+		debug("       time:"+toString(mirror_iterator->second->dld_time));
+		debug("       size:"+toString(mirror_iterator->second->dld_size));
+		debug(" multiplier:"+toString(mirror_iterator->second->multiplier));
+		debug("  criterion:"+toString(mirror_iterator->second->get_laziness_criterion()));
+		return mirror_iterator->second;
+	}
+}
+
+
+string strip_mirror_name(string path){
+	string mirror_name;
+	mirror_name=path.substr(0,path.find("/",(path.find("://",0)+3)));
+	return mirror_name;
+}
+
+#endif
\ No newline at end of file
diff --git a/segget/segget.cpp b/segget/segget.cpp
index 2ea2045..492f7c1 100644
--- a/segget/segget.cpp
+++ b/segget/segget.cpp
@@ -54,13 +54,13 @@ int choose_segment(uint connection_num){
 	uint distfile_num(0);
 	uint segment_num(0);
 	while (pkg_num<stats.pkg_count){
-		debug("pkg_num:"+toString(pkg_num));
+//		debug("pkg_num:"+toString(pkg_num));
 		while(distfile_num<Ppkg_array[pkg_num]->distfile_count){
 			if (not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->downloaded)){
-				debug("	distfile_num:"+toString(distfile_num));
+//				debug("	distfile_num:"+toString(distfile_num));
 				if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->active_connections_num<settings.max_connection_num_per_distfile)
 					while (segment_num<Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->segments_count){
-						debug("		segment_num:"+toString(segment_num));
+//						debug("		segment_num:"+toString(segment_num));
 						//	segments_in_progress[connection_num]=
 						//	if not(Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->get_segment_downloaded_status(segment_num);
 						if (Ppkg_array[pkg_num]->Pdistfile_list[distfile_num]->dn_segments[segment_num].status==WAITING){
@@ -142,16 +142,21 @@ int download_pkgs(){
 				Tsegment *current_segment;
 				CURL *e = msg->easy_handle;
 				curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &current_segment);
-				string result_msg_text="RESULT:"+toString(msg->data.result)+" "+curl_easy_strerror(msg->data.result)+" while downloading segment";
+				uint result=msg->data.result;
+				string result_msg_text="RESULT:"+toString(result)+" "+curl_easy_strerror(msg->data.result)+" while downloading segment";
 				msg_status1(current_segment->connection_num,current_segment->segment_num,result_msg_text);
 				curl_multi_remove_handle(cm, e);
 				fclose(current_segment->segment_file);
+				Tmirror *Pcurr_mirror=find_mirror(strip_mirror_name(current_segment->url));
+				time_t now_time = time((time_t *)NULL);
 				Tdistfile* prnt_distfile;
 				prnt_distfile=(Tdistfile*)current_segment->parent_distfile;
 				prnt_distfile->active_connections_num--;
-				if (msg->data.result){
+				if (result!=0){
 					// error -> start downloading again
-					msg_status2(current_segment->connection_num, " Failed download "+current_segment->file_name);
+					msg_status2(current_segment->connection_num, toString(result)+"]- Failed download "+current_segment->file_name);
+					debug(toString(result)+"]- Failed download "+current_segment->url);
+					Pcurr_mirror->stop(now_time-connection_array[current_segment->connection_num].start_time,0);
 					if (current_segment->try_num>=settings.max_tries){
 						current_segment->status=FAILED;
 						error_log("Segment:"+current_segment->file_name+" has reached max_tries limit - segment.status set to FAILED");
@@ -163,6 +168,8 @@ int download_pkgs(){
 				}else{
 					// no error => count this one and start new
 					log("Succesfully downloaded "+current_segment->file_name+" on connection#"+toString(current_segment->connection_num));
+					debug(" Successful download "+current_segment->url);
+					Pcurr_mirror->stop(now_time-connection_array[current_segment->connection_num].start_time,current_segment->segment_size);
 					current_segment->status=DOWNLOADED;
 					prnt_distfile->inc_dld_segments_count(current_segment);
 				};
diff --git a/segget/segment.cpp b/segget/segment.cpp
index bc4d960..87067b9 100644
--- a/segget/segment.cpp
+++ b/segget/segment.cpp
@@ -94,6 +94,7 @@ void Tsegment::prepare_for_connection(CURLM *cm, uint con_num, uint distfile_num
 	status=DOWNLOADING;
 	downloaded_bytes=0;
 	connection_num=con_num;
+	connection_array[con_num].start_time=time((time_t *)NULL);
 	url=segment_url;
 	try_num++;
 	add_easy_handle_to_multi(cm);
diff --git a/segget/settings.cpp b/segget/settings.cpp
index cbb62e6..b209814 100644
--- a/segget/settings.cpp
+++ b/segget/settings.cpp
@@ -10,6 +10,7 @@ using namespace std;
 class Tsettings{
 	private:
 	public:
+		uint forgetness;
 	//folders
 		string distfiles_dir;
 		string segments_dir;
@@ -28,7 +29,10 @@ class Tsettings{
 		uint low_connection_speed_time;
 		uint max_connection_speed;
 		string bind_interface;
+	//mirrors
+		uint max_connections_num_per_mirror;
 		Tsettings():
+			forgetness (5),
 		//folders
 			distfiles_dir("./distfiles"),
 			segments_dir("./tmp"),
@@ -46,7 +50,9 @@ class Tsettings{
 			low_connection_speed_limit(1000),
 			low_connection_speed_time(10),
 			max_connection_speed(0),
-			bind_interface("none")
+			bind_interface("none"),
+		//mirrors
+			max_connections_num_per_mirror(2)
 			{};
 		void set_resume(bool resume_setting){resume_on=resume_setting;};
 		bool get_resume(){return resume_on;};
@@ -73,6 +79,8 @@ void Tsettings::load_from_conf_file(){
 	conf.set(low_connection_speed_time,			"connections",		"low_connection_speed_time");
 	conf.set(max_connection_speed,				"connections",		"max_connection_speed");
 	conf.set(bind_interface,					"connections",		"bind_interface");
+
+	conf.set(max_connections_num_per_mirror,	"mirrors",			"max_connections_num_per_mirror");
 }
 
 Tsettings settings;
diff --git a/segget/stats.cpp b/segget/stats.cpp
index 487a59b..73bbda2 100644
--- a/segget/stats.cpp
+++ b/segget/stats.cpp
@@ -2,7 +2,6 @@
 #define __STATS_H__
 
 #include "connection.cpp"
-#include "checksum.cpp"
 
 class Tstats{
 	private: