new version of load and store that uses the wrapper lib

debug/common.hh

 #pragma once
-
-
-
-
 class metadata_files{
 public:
     string db,table;
@@ -158,10 +154,6 @@ void metadata_files::deserialize(std::string filename){
         int index = line.find(":");
         string head = line.substr(0,index);        
         if(head == "INDEX"){
-            //INIT HERE
-//            selected_field_types.push_back(vector<int>());
-//            selected_field_lengths.push_back(vector<int>());
-//            selected_field_names.push_back(vector<string>());
         }else if(head=="database"){
             set_db(line.substr(index+1));
         }else if(head=="table"){

debug/fast_insert.cc

+#include <iostream>
+#include <vector>
+#include <functional>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <main/Connect.hh>
+#include <main/rewrite_util.hh>
+#include <main/sql_handler.hh>
+#include <main/dml_handler.hh>
+#include <main/ddl_handler.hh>
+#include <main/CryptoHandlers.hh>
+#include <main/rewrite_main.hh>
+static std::string embeddedDir="/t/cryt/shadow";
+
+/*convert lex of insert into string*/
+static
+std::ostream&
+simple_insert(std::ostream &out, LEX &lex){
+        String s;
+        THD *t = current_thd;
+        const char* cmd = "INSERT";
+        out<<cmd<<" ";
+        lex.select_lex.table_list.first->print(t, &s, QT_ORDINARY);
+        out << "INTO " << s;
+        out << " values " << noparen(lex.many_values);
+        return out;
+}
+
+static
+std::string
+convert_insert(const LEX &lex)
+{
+    std::ostringstream o;
+    simple_insert(o,const_cast<LEX &>(lex));
+    return o.str();
+}
+
+
+/************************************************************************************************/
+
+
+/*encrypt one onion to get item. if the onion exists, then return directly*/
+static Item *
+my_encrypt_item_layers(const Item &i, onion o, const OnionMeta &om,
+                    const Analysis &a, uint64_t IV) {
+    assert(!RiboldMYSQL::is_null(i));
+    const auto &enc_layers = a.getEncLayers(om);
+    assert_s(enc_layers.size() > 0, "onion must have at least one layer");
+    const Item *enc = &i;
+    Item *new_enc = NULL;
+    for (const auto &it : enc_layers) {
+        new_enc = it->encrypt(*enc, IV);
+        assert(new_enc);
+        enc = new_enc;
+    }
+    assert(new_enc && new_enc != &i);
+    return new_enc;
+}
+
+
+/*encrypt the item to get a set of onions, in the parater l,salt is also added*/
+static
+void
+my_typical_rewrite_insert_type(const Item &i, const FieldMeta &fm,
+                            Analysis &a, std::vector<Item *> *l) {
+    const uint64_t salt = fm.getHasSalt() ? randomValue() : 0;
+    uint64_t IV = salt;
+    for (auto it : fm.orderedOnionMetas()) {
+        const onion o = it.first->getValue();
+        OnionMeta * const om = it.second;
+        l->push_back(my_encrypt_item_layers(i, o, *om, a, IV));
+    }
+    if (fm.getHasSalt()) {
+        l->push_back(new Item_int(static_cast<ulonglong>(salt)));
+    }
+}
+
+static
+void myRewriteInsertHelper(const Item &i, const FieldMeta &fm, Analysis &a,
+                           List<Item> *const append_list){
+    std::vector<Item *> l;
+    my_typical_rewrite_insert_type(i,fm,a,&l);
+    for (auto it : l) {
+        append_list->push_back(it);
+    }
+}
+
+static std::string getInsertResults(Analysis a,LEX* lex){
+        LEX *const new_lex = copyWithTHD(lex);
+        const std::string &table =
+            lex->select_lex.table_list.first->table_name;
+        const std::string &db_name =
+            lex->select_lex.table_list.first->db;
+        //from databasemeta to tablemeta.
+        const TableMeta &tm = a.getTableMeta(db_name, table);
+
+        //rewrite table name
+        new_lex->select_lex.table_list.first =
+            rewrite_table_list(lex->select_lex.table_list.first, a);
+
+        std::vector<FieldMeta *> fmVec;
+        std::vector<Item *> implicit_defaults;
+        
+        // No field list, use the table order.
+        assert(fmVec.empty());
+        std::vector<FieldMeta *> fmetas = tm.orderedFieldMetas();
+        fmVec.assign(fmetas.begin(), fmetas.end());
+
+        if (lex->many_values.head()) {
+            //start processing many values
+            auto it = List_iterator<List_item>(lex->many_values);
+            List<List_item> newList;
+            for (;;) {
+                List_item *const li = it++;
+                if (!li) {
+                    break;
+                }
+                List<Item> *const newList0 = new List<Item>();
+                if (li->elements != fmVec.size()) {
+                    TEST_TextMessageError(0 == li->elements
+                                         && NULL == lex->field_list.head(),
+                                          "size mismatch between fields"
+                                          " and values!");
+                } else {
+                    auto it0 = List_iterator<Item>(*li);
+                    auto fmVecIt = fmVec.begin();
+
+                    for (;;) {
+                        const Item *const i = it0++;
+                        assert(!!i == (fmVec.end() != fmVecIt));
+                        if (!i) {
+                            break;
+                        }
+                        //fetch values, and use fieldMeta to facilitate rewrite
+                        //every filed should be encrypted with onions of encryption
+                        myRewriteInsertHelper(*i, **fmVecIt, a, newList0);
+                        ++fmVecIt;
+                    }
+                    for (auto def_it : implicit_defaults) {
+                        newList0->push_back(def_it);
+                    }
+                }
+                newList.push_back(newList0);
+            }
+            new_lex->many_values = newList;
+        }
+        return convert_insert(*new_lex);
+}
+
+
+static void testInsertHandler(std::string query){
+    std::unique_ptr<Connect> e_conn(Connect::getEmbedded(embeddedDir));
+    std::unique_ptr<SchemaInfo> schema(new SchemaInfo());
+    std::function<DBMeta *(DBMeta *const)> loadChildren =
+        [&loadChildren, &e_conn](DBMeta *const parent) {
+            auto kids = parent->fetchChildren(e_conn);
+            for (auto it : kids) {
+                loadChildren(it);
+            }
+            return parent;
+        };
+    //load all metadata and then store it in schema
+    loadChildren(schema.get()); 
+    const std::unique_ptr<AES_KEY> &TK = std::unique_ptr<AES_KEY>(getKey(std::string("113341234")));
+    //just like what we do in Rewrite::rewrite,dispatchOnLex
+    Analysis analysis(std::string("tdb"),*schema,TK,
+                        SECURITY_RATING::SENSITIVE);
+    std::unique_ptr<query_parse> p;
+    p = std::unique_ptr<query_parse>(
+                new query_parse("tdb", query));
+    LEX *const lex = p->lex();
+
+    std::cout<<getInsertResults(analysis,lex)<<std::endl;
+}
+
+int
+main() {
+    char *buffer;
+    if((buffer = getcwd(NULL, 0)) == NULL){
+        perror("getcwd error");
+    }
+    embeddedDir = std::string(buffer)+"/shadow";
+    const std::string master_key = "113341234";
+    ConnectionInfo ci("localhost", "root", "letmein",3306);
+    SharedProxyState *shared_ps = 
+                     new SharedProxyState(ci, embeddedDir , master_key, determineSecurityRating());
+    assert(shared_ps!=NULL);
+    std::string query1 = "insert into child values(1,\"ZHAOYUN\")";
+    std::vector<std::string> querys{query1};
+
+    for(auto item:querys){
+        std::cout<<item<<std::endl;
+        testInsertHandler(item);
+        std::cout<<std::endl;
+    }
+    return 0;
+}

debug/load.cc

 #include "debug/load.hh"
-#include "debug/common.hh"
+#include "wrapper/common.hh"
 //get returnMeta
 //for each filed, we have a fieldmeta. we can chosse one onion under that field to construct a return meta.
 //in fact, a returnmeta can contain many fields.

debug/store.cc

 #include "debug/store.hh"
-#include "debug/common.hh"
+#include "wrapper/common.hh"
 static void write_meta(rawMySQLReturnValue& resraw,std::vector<transField> &res,string db,string table){
     metadata_files mf;
     mf.set_db_table(db,table);

debug/test_insert_convert.cc

+#include <iostream>
+#include <vector>
+#include <functional>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <main/Connect.hh>
+#include <main/rewrite_util.hh>
+#include <main/sql_handler.hh>
+#include <main/dml_handler.hh>
+#include <main/ddl_handler.hh>
+#include <main/CryptoHandlers.hh>
+#include <main/rewrite_main.hh>
+
+static std::string embeddedDir="/t/cryt/shadow";
+
+static void testInsertHandler(std::string query){
+    std::unique_ptr<query_parse> p;
+    p = std::unique_ptr<query_parse>(
+                new query_parse("tdb", query));
+    LEX *const lex = p->lex();
+    std::cout<<lexToQuery(*lex)<<std::endl;
+}
+
+int
+main() {
+    char *buffer;
+    if((buffer = getcwd(NULL, 0)) == NULL){
+        perror("getcwd error");
+    }
+    embeddedDir = std::string(buffer)+"/shadow";
+    const std::string master_key = "113341234";
+    ConnectionInfo ci("localhost", "root", "letmein",3306);
+    SharedProxyState *shared_ps = new SharedProxyState(ci, embeddedDir , master_key, determineSecurityRating());
+    assert(shared_ps!=NULL);
+    std::string query1("insert into child values(1,\"ZHAA\0\0OY\0\0UN\")",42);
+    std::vector<std::string> querys{query1};
+    for(auto item:querys){
+        std::cout<<item<<std::endl;
+        testInsertHandler(item);
+        std::cout<<std::endl;
+    }
+    return 0;
+}

debug/try_insert.cc

@@ -122,7 +122,7 @@ main() {
     ConnectionInfo ci("localhost", "root", "letmein",3306);
     SharedProxyState *shared_ps = new SharedProxyState(ci, embeddedDir , master_key, determineSecurityRating());
     assert(shared_ps!=NULL);
-    std::string query1 = "insert into student values(1,\"zhangfei\")";
+    std::string query1 = "insert into child values(1,\"zhangfei\")";
     std::vector<std::string> querys{query1};
     for(auto item:querys){
         std::cout<<item<<std::endl;

wrapper/Makefrag

+OBJDIRS	+= wrapper
+
+WRAPPER_SRCS := common.cc
+
+all: $(OBJDIR)/libwrapper.so
+
+WRAPPER_OBJS := $(patsubst %.cc,$(OBJDIR)/wrapper/%.o,$(WRAPPER_SRCS))
+
+$(OBJDIR)/libwrapper.so: $(WRAPPER_OBJS) \
+			 $(OBJDIR)/libedbcrypto.so \
+			 $(OBJDIR)/libedbutil.so \
+			 $(OBJDIR)/libedbparser.so \
+			 $(OBJDIR)/libcryptdb.so
+	$(CXX) -shared -g -o $@ $(WRAPPER_OBJS) $(LDFLAGS) $(LDRPATH) \
+	       -ledbcrypto -ledbutil -ledbparser -lntl -lcrypto -lcryptdb
+
+install: install_wrapper
+
+.PHONY: install_wrapper
+install_wrapper: $(OBJDIR)/libwrapper.so
+	install -m 644 $(OBJDIR)/libwrapper.so /usr/lib
+# vim: set noexpandtab:

wrapper/common.cc

+#include "wrapper/common.hh"
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fstream>
+
+using std::ifstream;
+
+string metadata_files::serialize_vec_int(string s,vector<int> vec_int){
+    s+=":";
+    for(auto item:vec_int){
+        s+=to_string(item)+=" ";
+    }
+    s.back()='\n';
+    return s;
+}
+string metadata_files::serialize_vec_str(string s,vector<string> vec_str){
+    s+=":";
+    for(auto item:vec_str){
+        s+=item+=" ";
+    }
+    s.back()='\n';
+    return s;
+}
+
+
+vector<string> metadata_files::string_to_vec_str(string line){
+    int start=0,next=0;
+    vector<string> tmp;
+    while((next=line.find(' ',start))!=-1){
+        string item = line.substr(start,next-start);
+        tmp.push_back(item);
+        start = next+1;
+    }
+    string item = line.substr(start);
+    tmp.push_back(item);
+    return tmp;
+}
+
+
+vector<int> metadata_files::string_to_vec_int(string line){
+    int start=0,next=0;
+    vector<int> tmp;
+    while((next=line.find(' ',start))!=-1){
+        string item = line.substr(start,next-start);
+        tmp.push_back(stoi(item));
+        start = next+1;
+    }
+    string item = line.substr(start);
+    tmp.push_back(stoi(item));
+    return tmp;
+}
+
+
+bool metadata_files::make_path(string directory){
+    struct stat st;
+    if(directory.size()==0||directory[0]=='/') return false;
+    if(directory.back()=='/') directory.pop_back();
+    int start = 0,next=0;
+    while(stat(directory.c_str(),&st)==-1&&next!=-1){
+        next = directory.find('/',start);
+        if(next!=-1){
+            string sub = directory.substr(0,next);
+            if(stat(sub.c_str(),&st)==-1)
+                mkdir(sub.c_str(),0700);
+            start =  next + 1;
+        }else{
+            mkdir(directory.c_str(),0700);
+        }
+    }
+    return true;
+}
+
+
+void metadata_files::serialize(){
+    FILE * localmeta = NULL;
+    string prefix = string("data/")+db+"/"+table;
+    make_path(prefix);
+    localmeta = fopen((prefix+"/metadata.data").c_str(),"w");
+
+    string s = string("database:")+db;
+    s+="\n";
+    fwrite(s.c_str(),1,s.size(),localmeta);
+    s = string("table:")+table;
+    s+="\n";
+    fwrite(s.c_str(),1,s.size(),localmeta);
+    for(unsigned int i=0;i<dec_onion_index.size();i++){
+        s=string("INDEX:")+to_string(i)+"\n";
+        fwrite(s.c_str(),1,s.size(),localmeta);
+        //then for each index, that is, for each plain field
+        auto field_types = selected_field_types[i];
+        s = serialize_vec_int("field_types",field_types);
+        fwrite(s.c_str(),1,s.size(),localmeta);
+
+        auto field_lengths = selected_field_lengths[i];        
+        s = serialize_vec_int("field_lengths",field_lengths);
+        fwrite(s.c_str(),1,s.size(),localmeta);
+
+        auto field_names = selected_field_names[i];
+        s = serialize_vec_str("field_names",field_names);
+        fwrite(s.c_str(),1,s.size(),localmeta);
+
+        auto get_has_salt = has_salt[i];
+        s = string("has_salt:")+get_has_salt+string("\n");
+        fwrite(s.c_str(),1,s.size(),localmeta);
+      
+        auto onion_index = to_string(dec_onion_index[i]);
+        s = string("onion_index:")+onion_index+string("\n");
+        fwrite(s.c_str(),1,s.size(),localmeta);
+    }
+    fclose(localmeta);
+}
+
+void metadata_files::deserialize(string filename){
+    filename = string("data/")+db+"/"+table+"/"+filename;
+    ifstream infile(filename);
+    string line;
+    while(getline(infile,line)){
+        int index = line.find(":");
+        string head = line.substr(0,index);        
+        if(head == "INDEX"){
+        }else if(head=="database"){
+            set_db(line.substr(index+1));
+        }else if(head=="table"){
+            set_table(line.substr(index+1));
+        }else if(head=="field_types"){
+            string types = line.substr(index+1);
+            auto res = string_to_vec_int(types);
+            selected_field_types.push_back(res);
+        }else if(head=="field_lengths"){
+            string lengths = line.substr(index+1);
+            auto res = string_to_vec_int(lengths);
+            selected_field_lengths.push_back(res);
+        }else if(head=="field_names"){
+            string names = line.substr(index+1);
+            auto res = string_to_vec_str(names);
+            selected_field_names.push_back(res);
+        }else if(head=="has_salt"){
+            has_salt.push_back(line.substr(index+1));
+        }else if(head=="onion_index"){
+            dec_onion_index.push_back(stoi(line.substr(index+1)));
+        }else{
+            exit(-1);
+        }
+    }
+    infile.close();
+}
+

wrapper/common.hh

+#pragma once
+#include <stdlib.h>
+#include <stdio.h>
+#include <string>
+#include <map>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <vector>
+
+using std::cout;
+using std::cin;
+using std::endl;
+using std::vector;
+using std::string;
+using std::to_string;
+
+
+class metadata_files{
+public:
+    string db,table;
+    /*selected fields*/
+    vector<vector<int>> selected_field_types;
+    vector<vector<int>> selected_field_lengths;
+    vector<vector<string>> selected_field_names;
+    vector<string> has_salt;
+    vector<int> dec_onion_index;/*should be 0,1,2,3...*/
+    string serialize_vec_int(string s,vector<int> vec_int);
+    string serialize_vec_str(string s,vector<string> vec_str);
+    vector<string> string_to_vec_str(string line);
+    vector<int> string_to_vec_int(string line);
+    static bool make_path(string directory);
+public:
+    void set_db(string idb){db=idb;}
+    string get_db(){return db;}
+
+    void set_table(string itable){table=itable;}
+    string get_table(){return table;}
+
+    void set_db_table(string idb,string itable){db=idb;table=itable;}
+
+    void set_selected_field_types(vector<vector<int>> input){selected_field_types = input;}
+    vector<vector<int>> &get_selected_field_types(){return selected_field_types;};
+
+    void set_selected_field_lengths(vector<vector<int>> input){selected_field_lengths = input;}
+    vector<vector<int>> &get_selected_field_lengths(){return selected_field_lengths;}
+
+    void set_selected_field_names(vector<vector<string>> input){selected_field_names = input;}
+    vector<vector<string>> &get_selected_field_names(){return selected_field_names;}
+
+    void set_dec_onion_index(vector<int> input){dec_onion_index = input;}
+    vector<int> &get_dec_onion_index(){return dec_onion_index;}
+
+    void set_has_salt(vector<string> input){has_salt = input;}
+    vector<string> &get_has_salt(){return has_salt;}
+
+    void serialize();
+    void deserialize(string filename);
+};
+
+