use the same key for each layer

crypto/BasicCrypto.cc

@@ -457,7 +457,7 @@ getLayerKey(const AES_KEY * const mKey, string uniqueFieldName,
     if (l == SECLEVEL::OPEFOREIGN){
         return getKey(mKey,"foreignjoin",l);
     }
-    return getKey(mKey, uniqueFieldName, l);
+    return getKey(mKey, "uniqueFieldName", l);//disable layer key temporarily. shaoyiwen
 }
 
 

debug/fast_insert.cc

@@ -13,7 +13,7 @@
 #include <main/rewrite_main.hh>
 #include "wrapper/reuse.hh"
 
-std::map<onion,std::vector<std::string>> fileOnions;
+//std::map<onion,std::vector<std::string>> fileOnions;
 
 static std::string embeddedDir="/t/cryt/shadow";
 
@@ -72,14 +72,14 @@ my_typical_rewrite_insert_type(const Item &i, const FieldMeta &fm,
     for (auto it : fm.orderedOnionMetas()) {
         const onion o = it.first->getValue();
         OnionMeta * const om = it.second;      
-        if(o==oAGG){
-            std::vector<std::string> &tempFileVector = fileOnions[o];
-            std::string in(tempFileVector.back());
-            tempFileVector.pop_back();
-            l->push_back(MySQLFieldTypeToItem(static_cast<enum_field_types>(253),in));
-        }else{
+//        if(o==oAGG){
+//            std::vector<std::string> &tempFileVector = fileOnions[o];
+//            std::string in(tempFileVector.back());
+//            tempFileVector.pop_back();
+//            l->push_back(MySQLFieldTypeToItem(static_cast<enum_field_types>(253),in));
+//        }else{
             l->push_back(my_encrypt_item_layers(i, o, *om, a, IV));
-        }
+//        }
     }
     if (fm.getHasSalt()) {
         l->push_back(new Item_int(static_cast<ulonglong>(salt)));
@@ -185,6 +185,7 @@ static void testInsertHandler(std::string query,std::string db){
 }
 
 //fileOnions
+/*
 static 
 void initFileOnions(std::string filename,std::string dataPrefix="allTables/table_MGVGBABDMU/"){
     std::ifstream infile(filename);
@@ -202,7 +203,7 @@ void initFileOnions(std::string filename,std::string dataPrefix="allTables/table
     load_string_file(dataPrefix+onionAnnoName,res,256);
     std::reverse(res.begin(),res.end());
     fileOnions[oAGG] = std::move(res);
-}
+}*/
 
 int
 main(int argc,char**argv) {
@@ -217,7 +218,7 @@ main(int argc,char**argv) {
     }
     std::string db(argv[1]);
 
-    initFileOnions("conf/already.cnf");
+//    initFileOnions("conf/already.cnf");
 
     const std::string master_key = "113341234";
     ConnectionInfo ci("localhost", "root", "letmein",3306);

debug/fast_insert_use_hom.cc.back

+#include <iostream>
+#include <vector>
+#include <functional>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <main/Connect.hh>
+#include <main/rewrite_util.hh>
+#include <main/sql_handler.hh>
+#include <main/dml_handler.hh>
+#include <main/ddl_handler.hh>
+#include <main/CryptoHandlers.hh>
+#include <main/rewrite_main.hh>
+#include "wrapper/reuse.hh"
+
+std::map<onion,std::vector<std::string>> fileOnions;
+
+static std::string embeddedDir="/t/cryt/shadow";
+
+/*convert lex of insert into string*/
+static
+std::ostream&
+simple_insert(std::ostream &out, LEX &lex){
+        String s;
+        THD *t = current_thd;
+        const char* cmd = "INSERT";
+        out<<cmd<<" ";
+        lex.select_lex.table_list.first->print(t, &s, QT_ORDINARY);
+        out << "INTO " << s;
+        out << " values " << noparen(lex.many_values);
+        return out;
+}
+
+static
+std::string
+convert_insert(const LEX &lex)
+{
+    std::ostringstream o;
+    simple_insert(o,const_cast<LEX &>(lex));
+    return o.str();
+}
+
+
+/************************************************************************************************/
+
+/*encrypt one onion to get item. if the onion exists, then return directly*/
+static Item *
+my_encrypt_item_layers(const Item &i, onion o, const OnionMeta &om,
+                    const Analysis &a, uint64_t IV) {
+    assert(!RiboldMYSQL::is_null(i));
+    const auto &enc_layers = a.getEncLayers(om);
+    assert_s(enc_layers.size() > 0, "onion must have at least one layer");
+    const Item *enc = &i;
+    Item *new_enc = NULL;
+    for (const auto &it : enc_layers) {
+        new_enc = it->encrypt(*enc, IV);
+        assert(new_enc);
+        enc = new_enc;
+    }
+    assert(new_enc && new_enc != &i);
+    return new_enc;
+}
+
+
+/*encrypt the item to get a set of onions, in the parater l,salt is also added*/
+static
+void
+my_typical_rewrite_insert_type(const Item &i, const FieldMeta &fm,
+                            Analysis &a, std::vector<Item *> *l) {
+    const uint64_t salt = fm.getHasSalt() ? randomValue() : 0;
+    uint64_t IV = salt;
+    for (auto it : fm.orderedOnionMetas()) {
+        const onion o = it.first->getValue();
+        OnionMeta * const om = it.second;      
+        if(o==oAGG){
+            std::vector<std::string> &tempFileVector = fileOnions[o];
+            std::string in(tempFileVector.back());
+            tempFileVector.pop_back();
+            l->push_back(MySQLFieldTypeToItem(static_cast<enum_field_types>(253),in));
+        }else{
+            l->push_back(my_encrypt_item_layers(i, o, *om, a, IV));
+        }
+    }
+    if (fm.getHasSalt()) {
+        l->push_back(new Item_int(static_cast<ulonglong>(salt)));
+    }
+}
+
+static
+void myRewriteInsertHelper(const Item &i, const FieldMeta &fm, Analysis &a,
+                           List<Item> *const append_list){
+    std::vector<Item *> l;
+    my_typical_rewrite_insert_type(i,fm,a,&l);
+    for (auto it : l) {
+        append_list->push_back(it);
+    }
+}
+
+static std::string getInsertResults(Analysis a,LEX* lex){
+        LEX *const new_lex = copyWithTHD(lex);
+        const std::string &table =
+            lex->select_lex.table_list.first->table_name;
+        const std::string &db_name =
+            lex->select_lex.table_list.first->db;
+        //from databasemeta to tablemeta.
+        const TableMeta &tm = a.getTableMeta(db_name, table);
+
+        //rewrite table name
+        new_lex->select_lex.table_list.first =
+            rewrite_table_list(lex->select_lex.table_list.first, a);
+
+        std::vector<FieldMeta *> fmVec;
+        std::vector<Item *> implicit_defaults;
+        
+        // No field list, use the table order.
+        assert(fmVec.empty());
+        std::vector<FieldMeta *> fmetas = tm.orderedFieldMetas();
+        fmVec.assign(fmetas.begin(), fmetas.end());
+
+        if (lex->many_values.head()) {
+            //start processing many values
+            auto it = List_iterator<List_item>(lex->many_values);
+            List<List_item> newList;
+            for (;;) {
+                List_item *const li = it++;
+                if (!li) {
+                    break;
+                }
+                List<Item> *const newList0 = new List<Item>();
+                if (li->elements != fmVec.size()) {
+                    TEST_TextMessageError(0 == li->elements
+                                         && NULL == lex->field_list.head(),
+                                          "size mismatch between fields"
+                                          " and values!");
+                } else {
+                    auto it0 = List_iterator<Item>(*li);
+                    auto fmVecIt = fmVec.begin();
+
+                    for (;;) {
+                        const Item *const i = it0++;
+                        assert(!!i == (fmVec.end() != fmVecIt));
+                        if (!i) {
+                            break;
+                        }
+                        //fetch values, and use fieldMeta to facilitate rewrite
+                        //every filed should be encrypted with onions of encryption
+                        myRewriteInsertHelper(*i, **fmVecIt, a, newList0);
+                        ++fmVecIt;
+                    }
+                    for (auto def_it : implicit_defaults) {
+                        newList0->push_back(def_it);
+                    }
+                }
+                newList.push_back(newList0);
+            }
+            new_lex->many_values = newList;
+        }
+        return convert_insert(*new_lex);
+}
+
+
+static void testInsertHandler(std::string query,std::string db){
+    std::unique_ptr<Connect> e_conn(Connect::getEmbedded(embeddedDir));
+    std::unique_ptr<SchemaInfo> schema(new SchemaInfo());
+    std::function<DBMeta *(DBMeta *const)> loadChildren =
+        [&loadChildren, &e_conn](DBMeta *const parent) {
+            auto kids = parent->fetchChildren(e_conn);
+            for (auto it : kids) {
+                loadChildren(it);
+            }
+            return parent;
+        };
+    //load all metadata and then store it in schema
+    loadChildren(schema.get()); 
+    const std::unique_ptr<AES_KEY> &TK = std::unique_ptr<AES_KEY>(getKey(std::string("113341234")));
+    //just like what we do in Rewrite::rewrite,dispatchOnLex
+    Analysis analysis(std::string(db),*schema,TK,
+                        SECURITY_RATING::SENSITIVE);
+    std::unique_ptr<query_parse> p;
+    p = std::unique_ptr<query_parse>(
+                new query_parse(db, query));
+    LEX *const lex = p->lex();
+
+    std::cout<<getInsertResults(analysis,lex)<<std::endl;
+}
+
+//fileOnions
+static 
+void initFileOnions(std::string filename,std::string dataPrefix="allTables/table_MGVGBABDMU/"){
+    std::ifstream infile(filename);
+    std::string line;
+    
+    std::string onionAnnoName;
+    while(getline(infile,line)){
+        int index = line.find(":");
+        std::string head = line.substr(0,index);
+        std::string next = line.substr(index+1);
+        onionAnnoName = next;
+    }
+
+    std::vector<std::string> res;
+    load_string_file(dataPrefix+onionAnnoName,res,256);
+    std::reverse(res.begin(),res.end());
+    fileOnions[oAGG] = std::move(res);
+}
+
+int
+main(int argc,char**argv) {
+    char *buffer;
+    if((buffer = getcwd(NULL, 0)) == NULL){
+        perror("getcwd error");
+    }
+    embeddedDir = std::string(buffer)+"/shadow";
+
+    if(argc!=2){
+        assert(0);        
+    }
+    std::string db(argv[1]);
+
+    initFileOnions("conf/already.cnf");
+
+    const std::string master_key = "113341234";
+    ConnectionInfo ci("localhost", "root", "letmein",3306);
+    SharedProxyState *shared_ps = 
+                     new SharedProxyState(ci, embeddedDir , master_key, determineSecurityRating());
+    assert(shared_ps!=NULL);
+    std::string query1 = "insert into child values(1,\"ZHAOYUN\"),(2,'XC'),(3,'KK')";
+    std::vector<std::string> querys{query1};
+    for(auto item:querys){
+        std::cout<<item<<std::endl;
+        testInsertHandler(item,db);
+        std::cout<<std::endl;
+    }
+    return 0;
+}

debug/generateHom.cc

+#include <iostream>
+#include <vector>
+#include <functional>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <main/Connect.hh>
+#include <main/rewrite_util.hh>
+#include <main/sql_handler.hh>
+#include <main/dml_handler.hh>
+#include <main/ddl_handler.hh>
+#include <main/CryptoHandlers.hh>
+
+int
+main() {
+    AES_KEY *mk = getKey("");
+    std::string realkey = getLayerKey(mk,"hehe",SECLEVEL::HOM);
+    std::unique_ptr<EncLayer> q(new HOM(realkey));
+    (void)q;
+
+    return 0;
+}

main/CryptoHandlers.cc

@@ -1479,6 +1479,10 @@ HOM::HOM(const Create_field &f, const std::string &seed_key)
     : seed_key(seed_key), sk(NULL), waiting(true)
 {}
 
+HOM::HOM(const std::string &seed_key)//added by shaoyiwen for testing purposes
+    : seed_key(seed_key), sk(NULL), waiting(true)
+{}
+
 HOM::HOM(unsigned int id, const std::string &serial)
     : EncLayer(id), seed_key(serial), sk(NULL), waiting(true)
 {}

main/CryptoHandlers.hh

@@ -87,6 +87,7 @@ protected:
 class HOM : public EncLayer {
 public:
     HOM(const Create_field &cf, const std::string &seed_key);
+    HOM(const std::string &seed_key);
 
     // serialize and deserialize
     std::string doSerialize() const {return seed_key;}

packages/tls/mysql_wrapper/backFieldsToFiles.cc

@@ -29,23 +29,60 @@ string createSelect(string database,string table){
 void backupselect(string query,string prefix){
     system((string("rm -rf ")+prefix).c_str());
     system((string("mkdir -p ")+prefix).c_str());
-    auto dbresult = con->execute(query);
-    DBResult * result = dbresult.get();
+    MYSQL *conn = con->get_conn();
+    if (mysql_query(conn,query.c_str())) {
+        assert(0);
+    }
+    MYSQL_RES *result = mysql_store_result(conn);
+
+    if (result == NULL) {
+        return;
+    }
+    int num_fields = mysql_num_fields(result);
+    if(num_fields==0) return;
+
+    vector<vector<string>> rows;
+    vector<string> fields;
+    vector<enum_field_types> types;
+    vector<vector<int>> lengths;
+ 
+    //get fields and types
+    MYSQL_FIELD *field;
+    for(int i=0;i<num_fields;i++){
+        field = mysql_fetch_field(result);
+        if(field!=NULL){
+            fields.push_back(field->name);
+            types.push_back(field->type);
+        }else{
+            std::cout<<"field error"<<std::endl;
+        }
+    }
 
-    vector<vector<string>> rows = result->getRows();
-    vector<enum_field_types> types = result->getTypes();
-    vector<string> fields = result->getFields();
-    vector<vector<int>> lengths = result->getLengths();
     int len = fields.size();
     vector<FILE*> files(len,NULL);
     for(int i=0;i<fields.size();i++){
         files[i] = fopen((prefix+fields[i]).c_str(),"a");
     }
 
-    
-    for(auto i=0u;i<rows.size();i++){
-        for(int j=0;j<rows[i].size();j++){
-            fwrite(rows[i][j].c_str(),1,lengths[i][j],files[j]);
+   
+    MYSQL_ROW row;
+    while ((row = mysql_fetch_row(result))) {
+        unsigned long * fieldLen = mysql_fetch_lengths(result);
+        vector<string> curRow;
+        vector<int> curLength;
+        for(int i = 0; i < num_fields; i++) {
+            if(row[i]==NULL){
+                curRow.push_back(string("NULL"));
+                curLength.push_back(0);
+            }else{
+                curRow.push_back(string(row[i],fieldLen[i]));
+                curLength.push_back(fieldLen[i]);
+            }
+        }
+        //rows.push_back(std::move(curRow));
+        //lengths.push_back(curLength);
+        for(int j=0;j<curRow.size();j++){
+            fwrite(curRow[j].c_str(),1,curLength[j],files[j]);
             fprintf(files[j],"\n");
         }
     }