try to add lowmemory load

debug/final_load_low_memory.cc

@@ -132,6 +132,8 @@ void initGfb(std::vector<FieldMetaTrans> &res,std::string db,std::string table){
     gfb.field_lengths = field_lengths;
     //then we should read the vector
     std::string prefix = std::string("data/")+db+"/"+table+"/";
+
+    unsigned long tupleNum=0;
     for(unsigned int i=0u; i<gfb.field_names.size(); i++) {
         std::string filename = prefix + gfb.field_names[i];
         std::vector<std::string> column;
@@ -140,12 +142,17 @@ void initGfb(std::vector<FieldMetaTrans> &res,std::string db,std::string table){
         }else{
             load_string_file(filename,column,gfb.field_lengths[i]);
         }
+        tupleNum = column.size();
         gfb.annoOnionNameToFileVector[gfb.field_names[i]] = std::move(column);
     }
     //init another map
     for(unsigned int i=0;i<gfb.field_names.size();i++){
         gfb.annoOnionNameToType[gfb.field_names[i]] = gfb.field_types[i];
     }
+    //extra transformation. transform rows to item*
+    for(unsigned int i=0;i<gfb.field_names.size();i++){
+        gfb.annoOnionNameToItemVector[gfb.field_names[i]] = itemNullVector(tupleNum);
+    }
 }
 
 /*load file, decrypt, and then return data plain fields in the type ResType*/
@@ -192,7 +199,7 @@ static ResType load_files_low_memory(std::string db, std::string table){
     for(unsigned int i=0;i<field_types.size();++i){
 	resraw.fieldTypes.push_back(static_cast<enum_field_types>(field_types[i]));
     }
-
+    //mem 91%
     ResType rawtorestype = rawMySQLReturnValue_to_ResType(false, &resraw);
     auto finalresults = decryptResults(rawtorestype,*rm);
     return std::move(finalresults);
@@ -259,10 +266,12 @@ int
 main(int argc, char* argv[]){
     init();
     create_embedded_thd(0);
+    std::string ip = "localhost";
     std::string db="tdb",table="student";
     if(argc==3){
-        db = std::string(argv[1]);
-        table = std::string(argv[2]);
+        ip = std::string(argv[1]);
+        db = std::string(argv[2]);
+        table = std::string(argv[3]);
     }
 
     std::unique_ptr<SchemaInfo> schema =  myLoadSchemaInfo(embeddedDir);

debug/final_store.cc

@@ -121,11 +121,10 @@ main(int argc, char* argv[]){
     std::string db="tdb",table="student";
     std::string ip="127.0.0.1";
     int port=3306;
-    if(argc==5){
+    if(argc==4){
         ip = std::string(argv[1]);
-        port = std::stoi(std::string(argv[2]));
-        db = std::string(argv[3]);
-        table = std::string(argv[4]);
+        db = std::string(argv[2]);
+        table = std::string(argv[3]);
     }
     init(ip,port);
     store(db,table);

debug/micro_init_str.cc

@@ -4,18 +4,19 @@ using std::string;
 
 int
 main(int argc,char ** argv) {
-    if(argc<2){
-        std::cout<<"expect 1 or 2 arguments"<<std::endl;
+    if(argc!=3){
+        std::cout<<"expect ip and length"<<std::endl;
         return 0;
     }
-    string length(argv[1]);
+    string ip(argv[1]);
+    string length(argv[2]);
 
     std::vector<string> create{
         "create database micro_db;",
         "use micro_db;",
         string("create table str_table(name varchar(")+length+"));"
     };
-    big_proxy b;  
+    big_proxy b("tdb",ip,"root","letmein",3306);
     for(auto item:create){
         b.go(item);
     }

debug/micro_insert_int.cc

+#include "main/big_proxy.hh"
+#include <vector>
+using std::string;
+
+int
+main(int argc,char ** argv) {
+    std::vector<string> create{
+        "use micro_db;",
+        "insert into int_table values(1),(2),(3),(4),(5);",
+    };
+    if(argc!=2){
+        std::cout<<"1:ip"<<std::endl;
+        return 0;
+    }
+    std::string ip = std::string(argv[1]);
+    big_proxy b("tdb",ip,"root","letmein",3306);
+    for(auto item:create){
+        b.go(item);
+    }
+    return 0;
+}

debug/micro_insert_str.cc

+#include "main/big_proxy.hh"
+#include <vector>
+using std::string;
+
+int
+main(int argc,char ** argv) {
+    if(argc!=2){
+        std::cout<<"expect ip"<<std::endl;
+        return 0;
+    }
+    string ip(argv[1]);
+
+    std::vector<string> create{
+        "use micro_db;",
+        "insert into str_table values('a'),('b'),('c'),('d'),('e');"
+    };
+    big_proxy b("tdb",ip,"root","letmein",3306);
+    for(auto item:create){
+        b.go(item);
+    }
+    return 0;
+}
+

debug/micro_select_int.cc

+#include "main/big_proxy.hh"
+#include <vector>
+using std::string;
+
+int
+main(int argc,char ** argv) {
+    std::vector<string> create{
+        "use micro_db;",
+        "select * from int_table;",
+    };
+    if(argc!=2){
+        std::cout<<"1:ip"<<std::endl;
+        return 0;
+    }
+    std::string ip = std::string(argv[1]);
+    big_proxy b("tdb",ip,"root","letmein",3306);
+    for(auto item:create){
+        b.go(item);
+    }
+    return 0;
+}

debug/micro_select_str.cc

+#include "main/big_proxy.hh"
+#include <vector>
+using std::string;
+
+int
+main(int argc,char ** argv) {
+    if(argc!=2){
+        std::cout<<"expect ip and length"<<std::endl;
+        return 0;
+    }
+    string ip(argv[1]);
+
+    std::vector<string> create{
+        "use micro_db;",
+        "select * from str_table;"
+    };
+    big_proxy b("tdb",ip,"root","letmein",3306);
+    for(auto item:create){
+        b.go(item);
+    }
+    return 0;
+}
+

wrapper/reuse.cc

@@ -438,6 +438,16 @@ void load_num_file(std::string filename,std::vector<std::string> &res){
     }
     infile.close();
 }
+/*
+void load_num_file(std::string filename,std::vector<Item> &res,enum_field_types intype){
+    std::ifstream infile(filename);
+    std::string line;     
+    while(std::getline(infile,line)){
+        res.pusb_back((void*)MySQLFieldTypeToItem(intype,line));
+    }
+    infile.close();
+}*/
+
 
 
 void load_string_file(std::string filename, std::vector<std::string> &res,unsigned long length){
@@ -450,6 +460,17 @@ void load_string_file(std::string filename, std::vector<std::string> &res,unsign
     close(fd);
 }
 
+/*
+void load_string_file(std::string filename,std::vector<void*> &res,unsigned long length,enum_field_types intype){
+    char *buf = new char[length];
+    int fd = open(filename.c_str(),O_RDONLY);
+    if(fd==-1) assert(0);//reading from -1 may cause errors
+    while(read(fd,buf,length)!=0){
+        res.push_back((void*)MySQLFieldTypeToItem(intype,std::string(buf,length)));
+    }
+    close(fd);
+}
+*/
 
 std::ostream&
 insertManyValues(std::ostream &out,List<List_item> &newList){

wrapper/reuse.hh

@@ -167,6 +167,7 @@ struct fullBackUp{
     std::vector<int> field_types;
     std::vector<int> field_lengths;
     std::map<std::string,std::vector<std::string>> annoOnionNameToFileVector;//field name to vector of string
+    std::map<std::string,std::vector<Item*>> annoOnionNameToItemVector;
     std::map<std::string,int> annoOnionNameToType;
 };
 
@@ -184,3 +185,11 @@ getIntItem(int i);
 std::unique_ptr<Item>
 getStringItem(std::string s);
 
+
+//Item* do not work
+//void load_num_file(std::string filename,std::vector<Item> &res,enum_field_types intype);
+
+//void load_string_file(std::string filename,std::vector<Item> &res,unsigned long length,enum_field_types intype);
+
+
+