able to use onionindex in load and store

debug/common.hh

 #pragma once
 
-/*class metadata_file{
+
+
+
+class metadata_files{
+public:
     string db,table;
-    int num_of_fields;
-    vector<string> field_types;
-    vector<int> field_lengths;
-    vector<string> field_names;
-    vector<int> choosen_onions;
+    /*selected fields*/
+    vector<vector<int>> selected_field_types;
+    vector<vector<int>> selected_field_lengths;
+    vector<vector<string>> selected_field_names;
+    vector<string> has_salt;
+    vector<int> dec_onion_index;/*should be 0,1,2,3...*/
+    std::string serialize_vec_int(std::string s,vector<int> vec_int);
+    std::string serialize_vec_str(std::string s,vector<string> vec_str);
+    vector<string> string_to_vec_str(string line);
+    vector<int> string_to_vec_int(string line);
+    static bool make_path(std::string directory);
 public:
     void set_db(std::string idb){db=idb;}
     std::string get_db(){return db;}
+
     void set_table(std::string itable){table=itable;}
     std::string get_table(){return table;}
-    void set_db_table(std::string idb,std::string itable){db=idb;table=itable;}
-    void set_num_of_fields(int num){num_of_fields = num;}
-    int get_num_of_fields(){return num_of_fields;}
-    void set_field_types(vector<string> input){field_types = input;}
-    std::vector<std::string> & get_field_types(){return field_types;}
-    void set_field_lengths(vector<int> input){field_lengths = input;}
-    std::vector<int> & get_field_lengths(){return field_lengths;}
-    void set_field_names(vector<string> input){field_names = input;}
-    std::vector<std::string> & get_field_names(){return field_names;}
-    void set_choosen_onions(vector<int> input){choosen_onions = input;}
-    std::vector<int>& get_choosen_onions(){return choosen_onions;}
-    void serilize();
-    void deserialize(std::string filename);
-    void show();
-    static bool make_path(std::string directory){
-        struct stat st;
-        if(directory.size()==0||directory[0]=='/') return false;
-        if(directory.back()=='/') directory.pop_back();
-        int start = 0,next=0;
-        while(stat(directory.c_str(),&st)==-1&&next!=-1){
-            next = directory.find('/',start);
-            if(next!=-1){
-                string sub = directory.substr(0,next);
-                if(stat(sub.c_str(),&st)==-1)
-                    mkdir(sub.c_str(),0700);
-                start =  next + 1;
-            }else{
-                mkdir(directory.c_str(),0700);
-            }
-        }
-        return true;
-    }
-};
-
-void metadata_file::serilize(){
-    FILE * localmeta = NULL;
-    string prefix = string("data/")+db+"/"+table;
-    make_path(prefix);
-    localmeta = fopen((prefix+"/metadata.data").c_str(),"w");
-    string s = string("database:")+db;
-    s+="\n";
-    fwrite(s.c_str(),1,s.size(),localmeta);
-    s = string("table:")+table;
-    s+="\n";
-    fwrite(s.c_str(),1,s.size(),localmeta);
-
-    s = string("num_of_fields:")+to_string(num_of_fields)+"\n";
-    fwrite(s.c_str(),1,s.size(),localmeta);    
-
-    s = string("field_types:");
-    for(auto item:field_types){
-        s+=item+=" ";
-    }
-    s.back()='\n';
-    fwrite(s.c_str(),1,s.size(),localmeta);    
 
-    s = string("field_lengths:");
-    for(auto item : field_lengths){
-        s+=to_string(item)+=" ";
-    }
-    s.back()='\n';
-    fwrite(s.c_str(),1,s.size(),localmeta);
-
-    s = string("field_names:");
-    for(auto item : field_names){
-        s+=item+=" ";
-    }
-    s.back()='\n';
-    fwrite(s.c_str(),1,s.size(),localmeta);
+    void set_db_table(std::string idb,std::string itable){db=idb;table=itable;}
 
-    s = string("choosen_onions:");
-    for(auto item : choosen_onions){
-        s+=to_string(item)+=" ";
-    }
-    s.back()='\n';
-    fwrite(s.c_str(),1,s.size(),localmeta);
-    fclose(localmeta);
-}
+    void set_selected_field_types(vector<vector<int>> input){selected_field_types = input;}
+    vector<vector<int>> &get_selected_field_types(){return selected_field_types;};
 
-void metadata_file::deserialize(std::string filename){
-    filename = string("data/")+db+"/"+table+"/"+filename;
-    std::ifstream infile(filename);
-    string line;
-    while(std::getline(infile,line)){
-        int index = line.find(":");
-        string head = line.substr(0,index);
-        if(head=="database"){
-            set_db(line.substr(index+1));
-        }else if(head=="table"){
-            set_table(line.substr(index+1));
-        }else if(head=="num_of_fields"){
-            set_num_of_fields(std::stoi(line.substr(index+1)));
-        }else if(head=="field_types"){
-            string types = line.substr(index+1);
-            int start=0,next=0;
-            std::vector<std::string> tmp;
-            while((next=types.find(' ',start))!=-1){
-                string item = types.substr(start,next-start);
-                tmp.push_back(item);
-                start = next+1;
-            }
-            string item = types.substr(start);
-            tmp.push_back(item);
-            set_field_types(tmp);
-        }else if(head=="field_lengths"){
-            string lengths = line.substr(index+1);
-            int start=0,next=0;
-            std::vector<int> tmp;
-            while((next=lengths.find(' ',start))!=-1){
-                string item = lengths.substr(start,next-start);
-                tmp.push_back(std::stoi(item));
-                start = next+1;
-            }
-            string item = lengths.substr(start);
-            tmp.push_back(std::stoi(item));
-            set_field_lengths(tmp);
-        }else if(head=="field_names"){
-            std::vector<std::string> tmp;
-            string names = line.substr(index+1);
-            int start=0,next=0;
-            while((next=names.find(' ',start))!=-1){
-                string item = names.substr(start,next-start);
-                tmp.push_back(item);
-                start = next+1;
-            }
-            string item = names.substr(start);
-            tmp.push_back(item);
-            set_field_names(tmp);
-        }else if(head=="choosen_onions"){
-            std::vector<int> tmp;
-            string c_onions = line.substr(index+1);
-            int start=0,next=0;
-            while((next=c_onions.find(' ',start))!=-1){
-                string item = c_onions.substr(start,next-start);
-                tmp.push_back(std::stoi(item));
-                start = next+1;
-            }
-            string item = c_onions.substr(start);
-            tmp.push_back(std::stoi(item));
-            set_choosen_onions(tmp);
-        }    
-    }
-    infile.close();
-}
+    void set_selected_field_lengths(vector<vector<int>> input){selected_field_lengths = input;}
+    vector<vector<int>> &get_selected_field_lengths(){return selected_field_lengths;}
 
-void metadata_file::show(){
-    cout<<db<<endl;
-    cout<<table<<endl;
-    cout<<num_of_fields<<endl;
-    for(auto item:field_types){
-        cout<<item<<"\t";
-    }
-    cout<<endl;
-    for(auto item:field_lengths){
-        cout<<item<<"\t";
-    }
-	cout<<endl;
-    for(auto item:field_names){
-        cout<<item<<"\t";
-    }
-	cout<<endl;
-    for(auto item:choosen_onions){
-       cout<<item<<"\t";
-    }
-    cout<<endl;
-}
-*/
+    void set_selected_field_names(vector<vector<string>> input){selected_field_names = input;}
+    vector<vector<string>> &get_selected_field_names(){return selected_field_names;}
 
+    void set_dec_onion_index(vector<int> input){dec_onion_index = input;}
+    vector<int> &get_dec_onion_index(){return dec_onion_index;}
 
+    void set_has_salt(vector<string> input){has_salt = input;}
+    vector<string> &get_has_salt(){return has_salt;}
 
-/*******************************************************************************************************************/
+    void serialize();
+    void deserialize(std::string filename);
+};
 
-class metadata_files{
-public:
-    string db,table;
-    /*selected fields*/
-    vector<vector<int>> selected_field_types;
-    vector<vector<int>> selected_field_lengths;
-    vector<vector<string>> selected_field_names;
-    vector<string> has_salt;
-    vector<int> dec_onion_index;/*should be 0,1,2,3...*/
-    std::string serialize_vec_int(std::string s,vector<int> vec_int){
+std::string metadata_files::serialize_vec_int(std::string s,vector<int> vec_int){
     s+=":";
     for(auto item:vec_int){
         s+=to_string(item)+=" ";
     }
     s.back()='\n';
     return s;
-    }
-    std::string serialize_vec_str(std::string s,vector<string> vec_str){
+}
+std::string metadata_files::serialize_vec_str(std::string s,vector<string> vec_str){
     s+=":";
     for(auto item:vec_str){
         s+=item+=" ";
     }
     s.back()='\n';
     return s;
-    }
-    vector<string> string_to_vec_str(string line){
+}
+
+
+vector<string> metadata_files::string_to_vec_str(string line){
     int start=0,next=0;
     std::vector<std::string> tmp;
     while((next=line.find(' ',start))!=-1){
@@ -220,9 +74,10 @@ public:
     string item = line.substr(start);
     tmp.push_back(item);
     return tmp;
-    }
+}
+
 
-    vector<int> string_to_vec_int(string line){
+vector<int> metadata_files::string_to_vec_int(string line){
     int start=0,next=0;
     std::vector<int> tmp;
     while((next=line.find(' ',start))!=-1){
@@ -233,22 +88,10 @@ public:
     string item = line.substr(start);
     tmp.push_back(std::stoi(item));
     return tmp;
-    }
-public:
-    void set_db(std::string idb){db=idb;}
-    std::string get_db(){return db;}
-    void set_table(std::string itable){table=itable;}
-    std::string get_table(){return table;}
-    void set_db_table(std::string idb,std::string itable){db=idb;table=itable;}
-    void set_selected_field_types(vector<vector<int>> input){selected_field_types = input;}
-    void set_selected_field_lengths(vector<vector<int>> input){selected_field_lengths = input;}
-    void set_selected_field_names(vector<vector<string>> input){selected_field_names = input;}
-    void set_dec_onion_index(vector<int> input){dec_onion_index = input;}
-    void set_has_salt(vector<string> input){has_salt = input;}
-    void serialize();
-    void deserialize(std::string filename);
+}
 
-    static bool make_path(std::string directory){
+
+bool metadata_files::make_path(std::string directory){
     struct stat st;
     if(directory.size()==0||directory[0]=='/') return false;
     if(directory.back()=='/') directory.pop_back();
@@ -265,8 +108,8 @@ public:
         }
     }
     return true;
-    }
-};
+}
+
 
 void metadata_files::serialize(){
     FILE * localmeta = NULL;
@@ -316,9 +159,9 @@ void metadata_files::deserialize(std::string filename){
         string head = line.substr(0,index);        
         if(head == "INDEX"){
             //INIT HERE
-            selected_field_types.push_back(vector<int>());
-            selected_field_lengths.push_back(vector<int>());
-            selected_field_names.push_back(vector<string>());
+//            selected_field_types.push_back(vector<int>());
+//            selected_field_lengths.push_back(vector<int>());
+//            selected_field_names.push_back(vector<string>());
         }else if(head=="database"){
             set_db(line.substr(index+1));
         }else if(head=="table"){

debug/load.cc

@@ -54,18 +54,62 @@ vector<T> flat_vec(vector<vector<T>> &input){
     }
     return res;
 }
+struct batch{
+    vector<string> field_names;
+    vector<int> field_types;
+    vector<int> field_lengths;
+};
 
-static vector<vector<string>> load_table_fields(metadata_files & input) {
+static
+batch get_batch(metadata_files & input,std::vector<transField> &tfms){
+    vector<vector<int>> selected_field_types = input.get_selected_field_types();
+    vector<vector<int>> selected_field_lengths = input.get_selected_field_lengths();
+    vector<vector<string>> selected_field_names = input.get_selected_field_names();
+    vector<int> dec_onion_index = input.get_dec_onion_index();
+    vector<string> has_salt = input.get_has_salt();
+   
+    vector<string> field_names;
+    vector<int> field_types;
+    vector<int> field_lengths;
+
+    for(auto i=0u;i<tfms.size();i++){
+         int index = dec_onion_index[i];
+         string dec_field_name = tfms[i].fields[index];
+         auto f =  find(selected_field_names[i].begin(),selected_field_names[i].end(),dec_field_name);
+         assert(f!=selected_field_names[i].end());
+         int j = f - selected_field_names[i].begin();
+         if(has_salt[i]==string("true")){
+             field_names.push_back(selected_field_names[i][j]);
+             field_types.push_back(selected_field_types[i][j]);
+             field_lengths.push_back(selected_field_lengths[i][j]);
+
+             field_names.push_back(selected_field_names[i].back());
+             field_types.push_back(selected_field_types[i].back());
+             field_lengths.push_back(selected_field_lengths[i].back());             
+         }else{
+             assert(1==2);
+         }
+    }
+    batch bt;
+    bt.field_names = field_names;
+    bt.field_types = field_types;
+    bt.field_lengths = field_lengths;
+    return bt;
+}
+
+#include <algorithm>
+static vector<vector<string>> load_table_fields(metadata_files & input,std::vector<transField> &tfms) {
     string db = input.get_db();
     string table = input.get_table();
     vector<vector<string>> res;
     string prefix = string("data/")+db+"/"+table+"/";
 
-    vector<string> datafiles;
-    auto field_names = flat_vec(input.selected_field_names);
-    auto field_types = flat_vec(input.selected_field_types);
-    auto field_lengths = flat_vec(input.selected_field_lengths);
+    auto bt = get_batch(input,tfms);
+    vector<string> field_names = bt.field_names;
+    vector<int> field_types = bt.field_types;
+    vector<int> field_lengths = bt.field_lengths;
  
+    vector<string> datafiles;  
     for(auto item:field_names){
         datafiles.push_back(prefix+item);
     }
@@ -87,37 +131,32 @@ static vector<vector<string>> load_table_fields(metadata_files & input) {
     return res;
 }
 
-
 static ResType load_files(std::string db="tdb", std::string table="student"){
     std::unique_ptr<SchemaInfo> schema =  myLoadSchemaInfo();
     //get all the fields in the tables.
     std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
     auto res = getTransField(fms);
-
     std::vector<enum_field_types> types;//Added
-    for(auto item:fms){
-        types.push_back(item->getSqlType());
-    }//Add new field form FieldMeta
-    if(types.size()==1){
-        //to be
-    }
     metadata_files res_meta = load_meta(db,table);
     std::shared_ptr<ReturnMeta> rm = getReturnMeta(fms,res);
-
     //why do we need this??
     create_embedded_thd(0);
-
     rawMySQLReturnValue resraw;
-
     //load fields in the stored file
-    vector<vector<string>> res_field = load_table_fields(res_meta);
+    vector<vector<string>> res_field = load_table_fields(res_meta,res);
     resraw.rowValues = res_field;
-    auto field_names = flat_vec(res_meta.selected_field_names);
-    auto field_types = flat_vec(res_meta.selected_field_types);
-    auto field_lengths = flat_vec(res_meta.selected_field_lengths);
+
+//    auto field_names = flat_vec(res_meta.selected_field_names);
+//    auto field_types = flat_vec(res_meta.selected_field_types);
+//    auto field_lengths = flat_vec(res_meta.selected_field_lengths);
+
+    auto bt = get_batch(res_meta,res);
+    vector<string> field_names = bt.field_names;
+    vector<int> field_types = bt.field_types;
+    vector<int> field_lengths = bt.field_lengths;
 
     resraw.fieldNames = field_names;
-    for(unsigned int i=0;i<field_types.size();++i) {
+    for(unsigned int i=0;i<field_types.size();++i){
 	resraw.fieldTypes.push_back(static_cast<enum_field_types>(field_types[i]));
     }
     ResType rawtorestype = MygetResTypeFromLuaTable(false, &resraw);

debug/load.hh

@@ -131,8 +131,7 @@ static
 ResType MygetResTypeFromLuaTable(bool isNULL,rawMySQLReturnValue *inRow = NULL,int in_last_insert_id = 0){
     std::vector<std::string> names;
     std::vector<enum_field_types> types;
-    std::vector<std::vector<Item *> > rows;
-
+    std::vector<std::vector<Item *>> rows;
     //return NULL restype
     if(isNULL){
         return ResType(true,0,0,std::move(names),
@@ -151,9 +150,7 @@ ResType MygetResTypeFromLuaTable(bool isNULL,rawMySQLReturnValue *inRow = NULL,i
             }
             rows.push_back(curTempRow);
         }
-        return ResType(true, 0 ,
-                               in_last_insert_id, std::move(names),
-                                   std::move(types), std::move(rows));
+        return ResType(true,0,in_last_insert_id,std::move(names),std::move(types),std::move(rows));
     }
 }
 
@@ -323,17 +320,12 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
             dec_names.push_back(rf.fieldCalled());
         }
     }
-
-
     const unsigned int real_cols = dec_names.size();
-
     std::vector<std::vector<Item *> > dec_rows(rows);
-
     //real cols depends on plain text names.
     for (unsigned int i = 0; i < rows; i++) {
         dec_rows[i] = std::vector<Item *>(real_cols);
     }
-
     //
     unsigned int col_index = 0;
     for (unsigned int c = 0; c < cols; c++) {
@@ -341,16 +333,13 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
         if (rf.getIsSalt()) {
             continue;
         }
-
         //the key is in fieldMeta
         FieldMeta *const fm = rf.getOLK().key;
-
         for (unsigned int r = 0; r < rows; r++) {
 	    //
             if (!fm || dbres.rows[r][c]->is_null()) {
                 dec_rows[r][col_index] = dbres.rows[r][c];
             } else {
-
                 uint64_t salt = 0;
                 const int salt_pos = rf.getSaltPosition();
                 //read salt from remote datab for descrypting.
@@ -360,7 +349,6 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
                     assert_s(!salt_item->null_value, "salt item is null");
                     salt = salt_item->value;
                 }
-
                  //specify fieldMeta, onion, and salt should be able to decrpyt
                 //peel onion
                 dec_rows[r][col_index] =
@@ -369,13 +357,10 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
         }
         col_index++;
     }
-
     std::vector<enum_field_types> types;
-
     for(auto item:dec_rows[0]){
         types.push_back(item->field_type());
     }
-
     //resType is used befor and after descrypting.
     return ResType(dbres.ok, dbres.affected_rows, dbres.insert_id,
                    std::move(dec_names),
@@ -383,40 +368,6 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
                    std::move(dec_rows));
 }
 
-
-struct meta_file{
-    string db,table;
-    int num_of_fields;
-    vector<string> field_types;
-    vector<int> field_lengths;
-    vector<string> field_names;
-    vector<int> choosen_onions;
-
-    void show(){
-        cout<<db<<endl;
-        cout<<table<<endl;
-        cout<<num_of_fields<<endl;
-        for(auto item:field_types){
-            cout<<item<<"\t";
-        }
-        cout<<endl;
-        for(auto item:field_lengths){
-            cout<<item<<"\t";
-        }
-
-	cout<<endl;
-        for(auto item:field_names){
-            cout<<item<<"\t";
-        }
-	cout<<endl;
-        for(auto item:choosen_onions){
-           cout<<item<<"\t";
-        }
-        cout<<endl;
-    }
-};
-
-
 static void init(){
     std::string client="192.168.1.1:1234";
     //one Wrapper per user.
@@ -464,11 +415,8 @@ static void construct_insert(rawMySQLReturnValue & str,std::string table,std::ve
         cur+="(";        
         for(unsigned int j=0u;j<str.rowValues[i].size();j++){
             if(IS_NUM(str.fieldTypes[j])) {
-//                cout<<str.fieldTypes[j]<<endl;
                 cur+=str.rowValues[i][j]+=",";
-//                cout<<"isnum"<<endl;
             }else{
-                //cur+=string("\"")+=str.rowValues[i][j]+="\",";
                 int len = str.rowValues[i][j].size();
                 mysql_real_escape_string(globalConn->get_conn(),globalEsp,
                     str.rowValues[i][j].c_str(),len);

debug/store.cc

 #include "debug/store.hh"
 #include "debug/common.hh"
-
-
-
 static void write_meta(rawMySQLReturnValue& resraw,std::vector<transField> &res,string db,string table){
     metadata_files mf;
     mf.set_db_table(db,table);
-
     vector<vector<int>> selected_field_types;
     vector<vector<int>> selected_field_lengths;
     vector<vector<string>> selected_field_names;
+    vector<vector<int>> selected_onion_index;
     vector<int> dec_onion_index;
     vector<string> has_salt;
+
+    unsigned int type_index=0u,length_index=0u;
+
     for(auto item:res){
         vector<int> field_types;
         vector<int> field_lengths;
-        vector<string> field_names=item.fields;
+        
+        vector<string> field_names;
+        //only choosen fields
+        for(auto i:item.choosenOnions){
+            field_names.push_back(item.fields[i]);
+        }
+        if(item.hasSalt){
+            field_names.push_back(item.fields.back());
+        }
+
         int onion_index = item.onionIndex;
-        for(auto tp:resraw.fieldTypes)
-            field_types.push_back(static_cast<int>(tp));
-        field_lengths = resraw.lengths;
+
+        for(unsigned int i=0u;i<field_names.size();i++){
+            field_types.push_back(static_cast<int>(resraw.fieldTypes[type_index]));
+            type_index++;
+        }
+//        field_lengths = resraw.lengths;
+        for(unsigned int i=0u;i<field_names.size();i++){
+            field_lengths.push_back(resraw.lengths[length_index]);
+            length_index++;
+        }
         if(item.hasSalt){
             has_salt.push_back("true");
         }else has_salt.push_back("false");
@@ -70,6 +86,7 @@ static void store(std::string db, std::string table){
     std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
     //transform the field so that selected onions can be used
     std::vector<transField> res = getTransField(fms);
+
     for(auto &item:res){
         (void)item;
         item.choosenOnions.push_back(0);
@@ -80,7 +97,6 @@ static void store(std::string db, std::string table){
     //write the tuples into files
     write_raw_data_to_files(resraw,res,db,table);
 }
-
 int
 main(int argc, char* argv[]){
     init();

debug/store.hh

@@ -60,7 +60,6 @@ struct transField{
     vector<int> choosenOnions;
     //used to construct return meta
     int onionIndex = 0;
-    int numOfOnions=0;
     //onions
     std::vector<std::string> fields;
     std::vector<onion> onions;
@@ -192,11 +191,6 @@ rawMySQLReturnValue executeAndGetResultRemote(Connect * curConn,std::string quer
     return myRaw;
 }
 
-
-
-
-
-
 //first step of back
 static 
 std::vector<FieldMeta *> 
@@ -217,9 +211,6 @@ getFieldMeta(SchemaInfo &schema,
      }
 }
 
-
-
-
 static std::unique_ptr<SchemaInfo> myLoadSchemaInfo() {
     std::unique_ptr<Connect> e_conn(Connect::getEmbedded(embeddedDir));
     std::unique_ptr<SchemaInfo> schema(new SchemaInfo());
@@ -283,7 +274,6 @@ std::string getTestQuery(SchemaInfo &schema, std::vector<transField> &tfds,
             res += item.originalFm->getSaltName()+" , ";
         }
     }
-
     res = res.substr(0,res.size()-2);
     res = res + "FROM `"+db+std::string("`.`")+annotablename+"`";
     return res;