able to use onionindex in load and store

debug/load.cc

@@ -54,18 +54,62 @@ vector<T> flat_vec(vector<vector<T>> &input){
     }
     return res;
 }
+struct batch{
+    vector<string> field_names;
+    vector<int> field_types;
+    vector<int> field_lengths;
+};
 
-static vector<vector<string>> load_table_fields(metadata_files & input) {
+static
+batch get_batch(metadata_files & input,std::vector<transField> &tfms){
+    vector<vector<int>> selected_field_types = input.get_selected_field_types();
+    vector<vector<int>> selected_field_lengths = input.get_selected_field_lengths();
+    vector<vector<string>> selected_field_names = input.get_selected_field_names();
+    vector<int> dec_onion_index = input.get_dec_onion_index();
+    vector<string> has_salt = input.get_has_salt();
+   
+    vector<string> field_names;
+    vector<int> field_types;
+    vector<int> field_lengths;
+
+    for(auto i=0u;i<tfms.size();i++){
+         int index = dec_onion_index[i];
+         string dec_field_name = tfms[i].fields[index];
+         auto f =  find(selected_field_names[i].begin(),selected_field_names[i].end(),dec_field_name);
+         assert(f!=selected_field_names[i].end());
+         int j = f - selected_field_names[i].begin();
+         if(has_salt[i]==string("true")){
+             field_names.push_back(selected_field_names[i][j]);
+             field_types.push_back(selected_field_types[i][j]);
+             field_lengths.push_back(selected_field_lengths[i][j]);
+
+             field_names.push_back(selected_field_names[i].back());
+             field_types.push_back(selected_field_types[i].back());
+             field_lengths.push_back(selected_field_lengths[i].back());             
+         }else{
+             assert(1==2);
+         }
+    }
+    batch bt;
+    bt.field_names = field_names;
+    bt.field_types = field_types;
+    bt.field_lengths = field_lengths;
+    return bt;
+}
+
+#include <algorithm>
+static vector<vector<string>> load_table_fields(metadata_files & input,std::vector<transField> &tfms) {
     string db = input.get_db();
     string table = input.get_table();
     vector<vector<string>> res;
     string prefix = string("data/")+db+"/"+table+"/";
 
-    vector<string> datafiles;
-    auto field_names = flat_vec(input.selected_field_names);
-    auto field_types = flat_vec(input.selected_field_types);
-    auto field_lengths = flat_vec(input.selected_field_lengths);
-
+    auto bt = get_batch(input,tfms);
+    vector<string> field_names = bt.field_names;
+    vector<int> field_types = bt.field_types;
+    vector<int> field_lengths = bt.field_lengths;
+ 
+    vector<string> datafiles;  
     for(auto item:field_names){
         datafiles.push_back(prefix+item);
     }
@@ -87,37 +131,32 @@ static vector<vector<string>> load_table_fields(metadata_files & input) {
     return res;
 }
 
-
 static ResType load_files(std::string db="tdb", std::string table="student"){
     std::unique_ptr<SchemaInfo> schema =  myLoadSchemaInfo();
     //get all the fields in the tables.
     std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
     auto res = getTransField(fms);
-
     std::vector<enum_field_types> types;//Added
-    for(auto item:fms){
-        types.push_back(item->getSqlType());
-    }//Add new field form FieldMeta
-    if(types.size()==1){
-        //to be
-    }
     metadata_files res_meta = load_meta(db,table);
     std::shared_ptr<ReturnMeta> rm = getReturnMeta(fms,res);
-
     //why do we need this??
     create_embedded_thd(0);
-
     rawMySQLReturnValue resraw;
-
     //load fields in the stored file
-    vector<vector<string>> res_field = load_table_fields(res_meta);
+    vector<vector<string>> res_field = load_table_fields(res_meta,res);
     resraw.rowValues = res_field;
-    auto field_names = flat_vec(res_meta.selected_field_names);
-    auto field_types = flat_vec(res_meta.selected_field_types);
-    auto field_lengths = flat_vec(res_meta.selected_field_lengths);
-    
+
+//    auto field_names = flat_vec(res_meta.selected_field_names);
+//    auto field_types = flat_vec(res_meta.selected_field_types);
+//    auto field_lengths = flat_vec(res_meta.selected_field_lengths);
+
+    auto bt = get_batch(res_meta,res);
+    vector<string> field_names = bt.field_names;
+    vector<int> field_types = bt.field_types;
+    vector<int> field_lengths = bt.field_lengths;
+
     resraw.fieldNames = field_names;
-    for(unsigned int i=0;i<field_types.size();++i) {
+    for(unsigned int i=0;i<field_types.size();++i){
 	resraw.fieldTypes.push_back(static_cast<enum_field_types>(field_types[i]));
     }
     ResType rawtorestype = MygetResTypeFromLuaTable(false, &resraw);

debug/load.hh

@@ -131,9 +131,8 @@ static
 ResType MygetResTypeFromLuaTable(bool isNULL,rawMySQLReturnValue *inRow = NULL,int in_last_insert_id = 0){
     std::vector<std::string> names;
     std::vector<enum_field_types> types;
-    std::vector<std::vector<Item *> > rows;
-
-    //return NULL restype 
+    std::vector<std::vector<Item *>> rows;
+    //return NULL restype
     if(isNULL){
         return ResType(true,0,0,std::move(names),
                       std::move(types),std::move(rows));
@@ -151,9 +150,7 @@ ResType MygetResTypeFromLuaTable(bool isNULL,rawMySQLReturnValue *inRow = NULL,i
             }
             rows.push_back(curTempRow);
         }
-        return ResType(true, 0 ,
-                               in_last_insert_id, std::move(names),
-                                   std::move(types), std::move(rows));
+        return ResType(true,0,in_last_insert_id,std::move(names),std::move(types),std::move(rows));
     }
 }
 
@@ -323,17 +320,12 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
             dec_names.push_back(rf.fieldCalled());
         }
     }
-
-
     const unsigned int real_cols = dec_names.size();
-
     std::vector<std::vector<Item *> > dec_rows(rows);
-
     //real cols depends on plain text names.
     for (unsigned int i = 0; i < rows; i++) {
         dec_rows[i] = std::vector<Item *>(real_cols);
     }
-
     //
     unsigned int col_index = 0;
     for (unsigned int c = 0; c < cols; c++) {
@@ -341,16 +333,13 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
         if (rf.getIsSalt()) {
             continue;
         }
-
         //the key is in fieldMeta
         FieldMeta *const fm = rf.getOLK().key;
-
         for (unsigned int r = 0; r < rows; r++) {
 	    //
             if (!fm || dbres.rows[r][c]->is_null()) {
                 dec_rows[r][col_index] = dbres.rows[r][c];
             } else {
-
                 uint64_t salt = 0;
                 const int salt_pos = rf.getSaltPosition();
                 //read salt from remote datab for descrypting.
@@ -360,7 +349,6 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
                     assert_s(!salt_item->null_value, "salt item is null");
                     salt = salt_item->value;
                 }
-
                  //specify fieldMeta, onion, and salt should be able to decrpyt
                 //peel onion
                 dec_rows[r][col_index] =
@@ -369,13 +357,10 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
         }
         col_index++;
     }
-
     std::vector<enum_field_types> types;
-
     for(auto item:dec_rows[0]){
         types.push_back(item->field_type());
     }
-
     //resType is used befor and after descrypting.
     return ResType(dbres.ok, dbres.affected_rows, dbres.insert_id,
                    std::move(dec_names),
@@ -383,40 +368,6 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
                    std::move(dec_rows));
 }
 
-
-struct meta_file{
-    string db,table;
-    int num_of_fields;
-    vector<string> field_types;
-    vector<int> field_lengths;
-    vector<string> field_names;
-    vector<int> choosen_onions;
-
-    void show(){
-        cout<<db<<endl;
-        cout<<table<<endl;
-        cout<<num_of_fields<<endl;
-        for(auto item:field_types){
-            cout<<item<<"\t";
-        }
-        cout<<endl;
-        for(auto item:field_lengths){
-            cout<<item<<"\t";
-        }
-
-	cout<<endl;
-        for(auto item:field_names){
-            cout<<item<<"\t";
-        }
-	cout<<endl;
-        for(auto item:choosen_onions){
-           cout<<item<<"\t";
-        }
-        cout<<endl;
-    }
-};
-
-
 static void init(){
     std::string client="192.168.1.1:1234";
     //one Wrapper per user.
@@ -464,11 +415,8 @@ static void construct_insert(rawMySQLReturnValue & str,std::string table,std::ve
         cur+="(";        
         for(unsigned int j=0u;j<str.rowValues[i].size();j++){
             if(IS_NUM(str.fieldTypes[j])) {
-//                cout<<str.fieldTypes[j]<<endl;
                 cur+=str.rowValues[i][j]+=",";
-//                cout<<"isnum"<<endl;
             }else{
-                //cur+=string("\"")+=str.rowValues[i][j]+="\",";
                 int len = str.rowValues[i][j].size();
                 mysql_real_escape_string(globalConn->get_conn(),globalEsp,
                     str.rowValues[i][j].c_str(),len);

debug/store.cc

 #include "debug/store.hh"
 #include "debug/common.hh"
-
-
-
 static void write_meta(rawMySQLReturnValue& resraw,std::vector<transField> &res,string db,string table){
     metadata_files mf;
     mf.set_db_table(db,table);
-
     vector<vector<int>> selected_field_types;
     vector<vector<int>> selected_field_lengths;
     vector<vector<string>> selected_field_names;
+    vector<vector<int>> selected_onion_index;
     vector<int> dec_onion_index;
     vector<string> has_salt;
+
+    unsigned int type_index=0u,length_index=0u;
+
     for(auto item:res){
         vector<int> field_types;
         vector<int> field_lengths;
-        vector<string> field_names=item.fields;
+        
+        vector<string> field_names;
+        //only choosen fields
+        for(auto i:item.choosenOnions){
+            field_names.push_back(item.fields[i]);
+        }
+        if(item.hasSalt){
+            field_names.push_back(item.fields.back());
+        }
+
         int onion_index = item.onionIndex;
-        for(auto tp:resraw.fieldTypes)
-            field_types.push_back(static_cast<int>(tp));
-        field_lengths = resraw.lengths;
+
+        for(unsigned int i=0u;i<field_names.size();i++){
+            field_types.push_back(static_cast<int>(resraw.fieldTypes[type_index]));
+            type_index++;
+        }
+//        field_lengths = resraw.lengths;
+        for(unsigned int i=0u;i<field_names.size();i++){
+            field_lengths.push_back(resraw.lengths[length_index]);
+            length_index++;
+        }
         if(item.hasSalt){
             has_salt.push_back("true");
         }else has_salt.push_back("false");
@@ -27,7 +43,7 @@ static void write_meta(rawMySQLReturnValue& resraw,std::vector<transField> &res,
         selected_field_types.push_back(field_types);
         selected_field_lengths.push_back(field_lengths);
         selected_field_names.push_back(field_names);
-        dec_onion_index.push_back(onion_index);               
+        dec_onion_index.push_back(onion_index);
     }
     mf.set_selected_field_types(selected_field_types);
     mf.set_selected_field_lengths(selected_field_lengths);
@@ -70,6 +86,7 @@ static void store(std::string db, std::string table){
     std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
     //transform the field so that selected onions can be used
     std::vector<transField> res = getTransField(fms);
+
     for(auto &item:res){
         (void)item;
         item.choosenOnions.push_back(0);
@@ -80,7 +97,6 @@ static void store(std::string db, std::string table){
     //write the tuples into files
     write_raw_data_to_files(resraw,res,db,table);
 }
-
 int
 main(int argc, char* argv[]){
     init();

debug/store.hh

@@ -60,7 +60,6 @@ struct transField{
     vector<int> choosenOnions;
     //used to construct return meta
     int onionIndex = 0;
-    int numOfOnions=0;
     //onions
     std::vector<std::string> fields;
     std::vector<onion> onions;
@@ -192,11 +191,6 @@ rawMySQLReturnValue executeAndGetResultRemote(Connect * curConn,std::string quer
     return myRaw;
 }
 
-
-
-
-
-
 //first step of back
 static 
 std::vector<FieldMeta *> 
@@ -217,9 +211,6 @@ getFieldMeta(SchemaInfo &schema,
      }
 }
 
-
-
-
 static std::unique_ptr<SchemaInfo> myLoadSchemaInfo() {
     std::unique_ptr<Connect> e_conn(Connect::getEmbedded(embeddedDir));
     std::unique_ptr<SchemaInfo> schema(new SchemaInfo());
@@ -283,7 +274,6 @@ std::string getTestQuery(SchemaInfo &schema, std::vector<transField> &tfds,
             res += item.originalFm->getSaltName()+" , ";
         }
     }
-
     res = res.substr(0,res.size()-2);
     res = res + "FROM `"+db+std::string("`.`")+annotablename+"`";
     return res;