new format of metafile

debug/common.hh

@@ -179,3 +179,165 @@ void metadata_file::show(){
 }
 
 
+/*******************************************************************************************************************/
+
+class metadata_files{
+    string db,table;
+    /*selected fields*/
+    vector<vector<int>> selected_field_types;
+    vector<vector<int>> selected_field_lengths;
+    vector<vector<string>> selected_field_names;
+    vector<string> has_salt;
+    vector<int> dec_onion_index;/*should be 0,1,2,3...*/    
+    std::string serialize_vec_int(std::string s,vector<int> vec_int){
+        s+=":";
+        for(auto item:vec_int){
+            s+=to_string(item)+=" ";
+        }
+        s.back()='\n';
+        return s;
+    }
+    std::string serialize_vec_str(std::string s,vector<string> vec_str){
+        s+=":";
+        for(auto item:vec_str){
+            s+=item+=" ";
+        }
+        s.back()='\n';
+        return s;
+    }
+    vector<string> string_to_vec_str(string line){
+        int start=0,next=0;
+        std::vector<std::string> tmp;
+        while((next=line.find(' ',start))!=-1){
+            string item = line.substr(start,next-start);
+            tmp.push_back(item);
+            start = next+1;
+        }
+        string item = line.substr(start);
+        tmp.push_back(item);
+        return tmp;
+    }
+
+    vector<int> string_to_vec_int(string line){
+        int start=0,next=0;
+        std::vector<int> tmp;
+        while((next=line.find(' ',start))!=-1){
+            string item = line.substr(start,next-start);
+            tmp.push_back(std::stoi(item));
+            start = next+1;
+        }
+        string item = line.substr(start);
+        tmp.push_back(std::stoi(item));
+        return tmp;
+    }
+public:
+    void set_db(std::string idb){db=idb;}
+    std::string get_db(){return db;}
+    void set_table(std::string itable){table=itable;}
+    std::string get_table(){return table;}
+    void set_db_table(std::string idb,std::string itable){db=idb;table=itable;}
+    void set_selected_field_types(vector<vector<int>> input){selected_field_types = input;}
+    void set_selected_field_lengths(vector<vector<int>> input){selected_field_lengths = input;}
+    void set_selected_field_names(vector<vector<string>> input){selected_field_names = input;}
+    void set_dec_onion_index(vector<int> input){dec_onion_index = input;}
+    void set_has_salt(vector<string> input){has_salt = input;}
+    void serialize();
+    void deserialize(std::string filename);
+
+    static bool make_path(std::string directory){
+        struct stat st;
+        if(directory.size()==0||directory[0]=='/') return false;
+        if(directory.back()=='/') directory.pop_back();
+        int start = 0,next=0;
+        while(stat(directory.c_str(),&st)==-1&&next!=-1){
+            next = directory.find('/',start);
+            if(next!=-1){
+                string sub = directory.substr(0,next);
+                if(stat(sub.c_str(),&st)==-1)
+                    mkdir(sub.c_str(),0700);
+                start =  next + 1;
+            }else{
+                mkdir(directory.c_str(),0700);
+            }
+        }
+        return true;
+    }
+};
+
+void metadata_files::serialize(){
+    FILE * localmeta = NULL;
+    string prefix = string("adata/")+db+"/"+table;
+    make_path(prefix);
+    localmeta = fopen((prefix+"/metadata.data").c_str(),"w");
+
+    string s = string("database:")+db;
+    s+="\n";
+    fwrite(s.c_str(),1,s.size(),localmeta);
+    s = string("table:")+table;
+    s+="\n";
+    fwrite(s.c_str(),1,s.size(),localmeta);
+    for(unsigned int i=0;i<dec_onion_index.size();i++){
+        s=string("INDEX:")+to_string(i)+"\n";
+        fwrite(s.c_str(),1,s.size(),localmeta);
+        //then for each index, that is, for each plain field
+        auto field_types = selected_field_types[i];
+        s = serialize_vec_int("field_types",field_types);
+        fwrite(s.c_str(),1,s.size(),localmeta);
+
+        auto field_lengths = selected_field_lengths[i];        
+        s = serialize_vec_int("field_lengths",field_lengths);
+        fwrite(s.c_str(),1,s.size(),localmeta);
+
+        auto field_names = selected_field_names[i];
+        s = serialize_vec_str("field_names",field_names);
+        fwrite(s.c_str(),1,s.size(),localmeta);
+
+        auto get_has_salt = has_salt[i];
+        s = string("has_salt:")+get_has_salt+string("\n");
+        fwrite(s.c_str(),1,s.size(),localmeta);
+      
+        auto onion_index = to_string(dec_onion_index[i]);
+        s = string("onion_index:")+onion_index+string("\n");
+        fwrite(s.c_str(),1,s.size(),localmeta);
+    }
+    fclose(localmeta);
+}
+
+void metadata_files::deserialize(std::string filename){
+    filename = string("adata/")+db+"/"+table+"/"+filename;
+    std::ifstream infile(filename);
+    string line;
+    while(std::getline(infile,line)){
+        int index = line.find(":");
+        string head = line.substr(0,index);        
+        if(head == "INDEX"){
+            //INIT HERE
+            selected_field_types.push_back(vector<int>());
+            selected_field_lengths.push_back(vector<int>());
+            selected_field_names.push_back(vector<string>());
+        }else if(head=="database"){
+            set_db(line.substr(index+1));
+        }else if(head=="table"){
+            set_table(line.substr(index+1));
+        }else if(head=="field_types"){
+            string types = line.substr(index+1);
+            auto res = string_to_vec_int(types);
+            selected_field_types.push_back(res);
+        }else if(head=="field_lengths"){
+            string lengths = line.substr(index+1);
+            auto res = string_to_vec_int(lengths);
+            selected_field_lengths.push_back(res);
+        }else if(head=="field_names"){
+            string names = line.substr(index+1);
+            auto res = string_to_vec_str(names);
+            selected_field_names.push_back(res);
+        }else if(head=="has_salt"){
+            has_salt.push_back(line.substr(index+1));
+        }else if(head=="onion_index"){
+            dec_onion_index.push_back(stoi(line.substr(index+1)));
+        }else{
+            exit(-1);
+        }
+    }
+    infile.close();
+}

debug/load.cc

@@ -18,7 +18,6 @@ std::shared_ptr<ReturnMeta> getReturnMeta(std::vector<FieldMeta*> fms, std::vect
     return myReturnMeta;
 }
 
-
 static metadata_file load_meta(string db="tdb", string table="student", string filename="metadata.data"){
     metadata_file mf;
     mf.set_db(db);
@@ -27,8 +26,6 @@ static metadata_file load_meta(string db="tdb", string table="student", string f
     return mf;
 }
 
-
-
 static void load_num(string filename,vector<string> &res){
     std::ifstream infile(filename);
     string line;
@@ -89,18 +86,20 @@ static ResType load_files(std::string db="tdb", std::string table="student"){
     if(types.size()==1){
         //to be
     }
-
     metadata_file res_meta = load_meta(db,table);
-
     for(unsigned int i=0;i<res_meta.get_choosen_onions().size();i++){
+        //choosen onion for each field
 	res[i].choosenOnions.push_back(res_meta.get_choosen_onions()[i]);
     }
+
     std::shared_ptr<ReturnMeta> rm = getReturnMeta(fms,res);
 
     //why do we need this??
     std::string backq = "show databases";
     executeAndGetResultRemote(globalConn,backq);
+
     rawMySQLReturnValue resraw2;
+
     //load fields in the stored file
     vector<vector<string>> res_field = load_table_fields(res_meta);
     resraw2.rowValues = res_field;
@@ -114,22 +113,17 @@ static ResType load_files(std::string db="tdb", std::string table="student"){
     return finalresults;
 }
 
-
-
 int
-main(int argc, char* argv[]) {
+main(int argc, char* argv[]){
     init();
     std::string db="tdb",table="student";
-
     globalEsp = (char*)malloc(sizeof(char)*5000);
-
     if(globalEsp==NULL){
         printf("unable to allocate esp\n");
         return 0;
     }
     /*load and decrypt*/
     ResType res =  load_files(db,table);
-
     /*transform*/
     rawMySQLReturnValue str;
     transform_to_rawMySQLReturnValue(str,res);
@@ -141,6 +135,6 @@ main(int argc, char* argv[]) {
     }
     free(globalEsp);
     /*the next step is to construct encrypted insert query*/
-
     return 0;
 }
+

debug/load.hh

@@ -259,7 +259,6 @@ static std::vector<transField> getTransField(std::vector<FieldMeta *> pfms){
             tf.numOfOnions++;
             tf.fields.push_back((ompair.second)->getAnonOnionName());
             tf.onions.push_back(ompair.first->getValue());
-            tf.originalOm.push_back(ompair.second);
         }
         if(pfm->getHasSalt()){
             tf.hasSalt=true;

debug/readmeta.cc

+#include <stdlib.h>
+#include <stdio.h>
+#include <string>
+#include <map>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <vector>
+#include <main/rewrite_main.hh>
+#include <main/rewrite_util.hh>
+using std::cout;
+using std::cin;
+using std::endl;
+using std::vector;
+using std::string;
+using std::to_string;
+#include "debug/common.hh"
+
+int main(){
+    metadata_files mf;
+    mf.set_db_table("tdb","student");
+    mf.deserialize("metadata.data");
+    return 0;
+}

debug/store.cc

 #include "debug/store.hh"
 #include "debug/common.hh"
 
-static void write_meta(rawMySQLReturnValue& resraw,string db,string table){
-    metadata_file mf;
+static void write_meta(rawMySQLReturnValue& resraw,std::vector<transField> &res,string db,string table){
+    metadata_files mf;
     mf.set_db_table(db,table);
-    mf.set_num_of_fields(resraw.fieldNames.size());
-    std::vector<std::string> temp;
-    for(auto item:resraw.fieldTypes){
-        temp.push_back(std::to_string(static_cast<int>(item)));
-    }
-    mf.set_field_types(temp);
-    mf.set_field_lengths(resraw.lengths);
-    mf.set_field_names(resraw.fieldNames);
-    mf.set_choosen_onions(resraw.choosen_onions);
-    mf.serilize();
-}
 
+    vector<vector<int>> selected_field_types;
+    vector<vector<int>> selected_field_lengths;
+    vector<vector<string>> selected_field_names;
+    vector<int> dec_onion_index;
+    vector<string> has_salt;
+    for(auto item:res){
+        vector<int> field_types;
+        vector<int> field_lengths;
+        vector<string> field_names=item.fields;
+        int onion_index = item.onionIndex;
+        for(auto tp:resraw.fieldTypes)
+            field_types.push_back(static_cast<int>(tp));
+        field_lengths = resraw.lengths;
+        if(item.hasSalt){
+            has_salt.push_back("true");
+        }else has_salt.push_back("false");
 
+        selected_field_types.push_back(field_types);
+        selected_field_lengths.push_back(field_lengths);
+        selected_field_names.push_back(field_names);
+        dec_onion_index.push_back(onion_index);               
+    }
+    mf.set_selected_field_types(selected_field_types);
+    mf.set_selected_field_lengths(selected_field_lengths);
+    mf.set_selected_field_names(selected_field_names);
+    mf.set_dec_onion_index(dec_onion_index);
+    mf.set_has_salt(has_salt);
+//    std::vector<std::string> temp;
+//    for(auto item:resraw.fieldTypes){
+//        temp.push_back(std::to_string(static_cast<int>(item)));
+//    }
+//    mf.set_field_types(temp);
+//    mf.set_field_lengths(resraw.lengths);
+//    mf.set_field_names(resraw.fieldNames);
+//    mf.set_choosen_onions(resraw.choosen_onions);
+//    mf.serilize();
+    mf.serialize();
+}
 static void write_row_data(rawMySQLReturnValue& resraw,string db, string table){
     vector<FILE*> data_files;
     string prefix = string("data/")+db+"/"+table+"/";
@@ -38,36 +64,28 @@ static void write_row_data(rawMySQLReturnValue& resraw,string db, string table){
         fclose(item);
     }
 }
-
-
 static
-void write_raw_data_to_files(rawMySQLReturnValue& resraw,string db,string table){
+void write_raw_data_to_files(rawMySQLReturnValue& resraw,std::vector<transField> &res ,string db,string table){
     //write metafiles
-    write_meta(resraw,db,table);
+    write_meta(resraw,res,db,table);
     //write datafiles
     write_row_data(resraw,db,table);
 }
-
 static void store(std::string db, std::string table){
     std::unique_ptr<SchemaInfo> schema =  myLoadSchemaInfo();
     //get all the fields in the tables
     std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
     //transform the field so that selected onions can be used
     std::vector<transField> res = getTransField(fms);
-
-    for(auto &item:res){
-        item.choosenOnions.push_back(0);
-    }
     //generate the backup query and then fetch the tuples
-//    std::shared_ptr<ReturnMeta> rm = getReturnMeta(fms,res);
-    std::string backq = getTestQuery(*schema,res,db,table);
-    rawMySQLReturnValue resraw =  executeAndGetResultRemote(globalConn,backq);
-
+    std::string backup_query = getTestQuery(*schema,res,db,table);
+    rawMySQLReturnValue resraw =  executeAndGetResultRemote(globalConn,backup_query);
     for(auto &item:res){
-        resraw.choosen_onions.push_back(item.choosenOnions[0]);
+        (void)item;
+        resraw.choosen_onions.push_back(0);
     }
     //write the tuples into files
-    write_raw_data_to_files(resraw,db,table);
+    write_raw_data_to_files(resraw,res,db,table);
 }
 
 int

debug/store.hh

@@ -114,7 +114,7 @@ struct transField{
     //onions
     std::vector<std::string> fields;
     std::vector<onion> onions;
-    std::vector<OnionMeta*>originalOm;
+//    std::vector<OnionMeta*>originalOm;
 };
 
 
@@ -245,16 +245,14 @@ static std::vector<transField> getTransField(std::vector<FieldMeta *> pfms){
     //for every field
     for(auto pfm:pfms){
         transField tf;
-	    tf.originalFm = pfm;
+	tf.originalFm = pfm;
         for(std::pair<const OnionMetaKey *, OnionMeta *> &ompair:pfm->orderedOnionMetas()){
-            tf.numOfOnions++;
             tf.fields.push_back((ompair.second)->getAnonOnionName());
             tf.onions.push_back(ompair.first->getValue());
-            tf.originalOm.push_back(ompair.second);
         }
         if(pfm->getHasSalt()){
             tf.hasSalt=true;
-	        tf.fields.push_back(pfm->getSaltName());
+	    tf.fields.push_back(pfm->getSaltName());
         }
         res.push_back(tf);
     }
@@ -271,7 +269,8 @@ std::string getTestQuery(SchemaInfo &schema, std::vector<transField> &tfds,
     //get databaseMeta, search in the map
     DatabaseMeta * dbm = schema.getChild(*dbmeta_key);
     const TableMeta & tbm = *((*dbm).getChild(IdentityMetaKey(table)));
-    std::string annotablename = tbm.getAnonTableName();    
+    std::string annotablename = tbm.getAnonTableName();
+
     //then a list of onion names
     for(auto item:tfds){
         for(auto index:item.choosenOnions){
@@ -282,6 +281,7 @@ std::string getTestQuery(SchemaInfo &schema, std::vector<transField> &tfds,
             res += item.originalFm->getSaltName()+" , ";
         }
     }
+
     res = res.substr(0,res.size()-2);
     res = res + "FROM `"+db+std::string("`.`")+annotablename+"`";
     return res;