able to use oAGG file

debug/backupFieldsToFiles.cc

+/*
+To make this work properly, you should at least make sure that the database tdb exists.
+*/
+#include <iostream>
+#include <vector>
+#include <functional>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <main/Connect.hh>
+#include <main/rewrite_util.hh>
+#include <main/sql_handler.hh>
+#include <main/dml_handler.hh>
+#include <main/ddl_handler.hh>
+#include <main/CryptoHandlers.hh>
+#include "wrapper/reuse.hh"
+
+using std::vector;
+using std::string;
+
+static std::string embeddedDir="/t/cryt/shadow";
+struct batch{
+    vector<vector<string>> rows;
+    vector<string> fields;
+    vector<enum_field_types> types;
+    vector<vector<int>> lengths;
+};
+
+
+static
+batch executeAndGetBatch(Connect* connect, std::string query){
+    MYSQL *conn = connect->get_conn();
+    batch bt;
+    if (mysql_query(conn,query.c_str())) {
+        assert(0);
+    }
+    MYSQL_RES *result = mysql_store_result(conn);
+    if (result == NULL) {
+        return bt;
+    }
+    
+    int num_fields = mysql_num_fields(result);
+    if(num_fields==0) return bt;
+
+    std::vector<std::vector<std::string>> rows;
+    std::vector<std::string> fields;
+    std::vector<enum_field_types> types;
+    std::vector<std::vector<int>> lengths;
+
+    MYSQL_FIELD *field;
+    for(int i=0;i<num_fields;i++){
+        field = mysql_fetch_field(result);
+        if(field!=NULL){
+            fields.push_back(field->name);
+            types.push_back(field->type);
+        }else{
+            std::cout<<"field error"<<std::endl;
+        }
+    }
+
+    MYSQL_ROW row;
+    while ((row = mysql_fetch_row(result))) {
+        unsigned long * fieldLen = mysql_fetch_lengths(result);
+        vector<string> curRow;
+        vector<int> curLength;
+        for(int i = 0; i < num_fields; i++) {
+            if(row[i]==NULL){
+                curRow.push_back(string("NULL"));
+                curLength.push_back(0);
+            }else{
+                curRow.push_back(string(row[i],fieldLen[i]));
+                curLength.push_back(fieldLen[i]);
+            }
+        }
+        rows.push_back(curRow);
+        lengths.push_back(curLength);
+    }
+
+    bt.rows = rows;
+    bt.fields = fields;
+    bt.lengths = lengths;
+    bt.types = types;
+    return bt;
+}
+
+
+static
+string createSelect(Connect*connect,string database,string table){
+    auto res = executeAndGetBatch(connect, string("SELECT * FROM `")+database+"`.`"+string(table)+"` LIMIT 1;");
+    vector<vector<string>> rows = res.rows;
+    vector<enum_field_types> types = res.types;
+    vector<string> fields = res.fields;
+    string head = "SELECT ";
+    for(unsigned int i=0u;i<types.size();i++){
+        if(IS_NUM(types[i])){
+            head += fields[i]+",";
+        }else{
+            head += fields[i]+",";
+        }
+    }
+    head[head.size()-1]=' ';
+    head += "FROM `"+database+"`.`"+table+"`";
+    return head;
+}
+
+static
+void backupselect(Connect*connect, string query,string prefix){
+    system((string("rm -rf ")+prefix).c_str());
+    system((string("mkdir -p ")+prefix).c_str());
+
+    MYSQL *conn = connect->get_conn();
+    if (mysql_query(conn,query.c_str())) {
+        assert(0);
+    }
+    MYSQL_RES *result = mysql_store_result(conn);
+
+    if (result == NULL) {
+        return;
+    }
+    unsigned int num_fields = mysql_num_fields(result);
+    if(num_fields==0) return;
+
+    vector<vector<string>> rows;
+    vector<string> fields;
+    vector<enum_field_types> types;
+    vector<vector<int>> lengths;
+
+    //get fields and types
+    MYSQL_FIELD *field;
+    for(unsigned int i=0u;i<num_fields;i++){
+        field = mysql_fetch_field(result);
+        if(field!=NULL){
+            fields.push_back(field->name);
+            types.push_back(field->type);
+        }else{
+            std::cout<<"field error"<<std::endl;
+        }
+    }
+    unsigned int len = fields.size();
+    vector<FILE*> files(len,NULL);
+    for(unsigned int i=0u;i<fields.size();i++){
+        files[i] = fopen((prefix+fields[i]).c_str(),"a");
+    }
+    MYSQL_ROW row;
+    while ((row = mysql_fetch_row(result))) {
+        unsigned long * fieldLen = mysql_fetch_lengths(result);
+        vector<string> curRow;
+        vector<int> curLength;
+        for(unsigned int i = 0u; i < num_fields; i++) {
+            if(row[i]==NULL){
+                curRow.push_back(string("NULL"));
+                curLength.push_back(0);
+            }else{
+                curRow.push_back(string(row[i],fieldLen[i]));
+                curLength.push_back(fieldLen[i]);
+            }
+        }
+        //rows.push_back(std::move(curRow));
+        //lengths.push_back(curLength);
+        for(unsigned int j=0u;j<curRow.size();j++){
+            fwrite(curRow[j].c_str(),1,curLength[j],files[j]);
+            if(IS_NUM(types[j]))
+                fprintf(files[j],"\n");
+        }
+    }
+    for(unsigned int i=0u;i<fields.size();i++){
+        fclose(files[i]);
+    }
+}
+
+
+static
+std::vector<std::string> getTables(Connect*connect, std::string db){
+    std::string query = std::string("SHOW TABLES IN ")+db;
+
+    batch bt = executeAndGetBatch(connect,query);
+    
+    std::vector<std::vector<std::string>> rows = bt.rows;
+    std::vector<enum_field_types> types = bt.types;
+    std::vector<std::string> fieldNames = bt.fields;
+    std::vector<std::string> res;
+    for(auto item:rows){
+        assert(item.size()==1);
+        res.push_back(item[0]);
+    }
+    return res;
+}
+
+int
+main(int argc,char** argv) {
+    char *buffer;
+    if((buffer = getcwd(NULL, 0)) == NULL){
+        perror("getcwd error");
+    }
+    embeddedDir = std::string(buffer)+"/shadow";
+    if(argc!=2){
+        assert(0);
+    }    
+    const std::string master_key = "113341234";
+    ConnectionInfo ci("localhost", "root", "letmein",3306);
+    SharedProxyState *shared_ps = 
+        new SharedProxyState(ci, embeddedDir , master_key, determineSecurityRating());
+    assert(shared_ps!=NULL);
+    std::string db = std::string(argv[1]);;
+
+    Connect *connect = new Connect(ci.server, ci.user, ci.passwd, ci.port);
+    connect->execute(string("use ")+db);
+    auto tablesPrefix = getTables(connect,db);    
+    for(auto item:tablesPrefix){
+        string query = createSelect(connect,db,item);
+        backupselect(connect,query,string("allTables/")+item+"/");
+    }
+    return 0;
+}

debug/fast_insert.cc

@@ -11,6 +11,10 @@
 #include <main/ddl_handler.hh>
 #include <main/CryptoHandlers.hh>
 #include <main/rewrite_main.hh>
+#include "wrapper/reuse.hh"
+
+std::map<onion,std::vector<std::string>> fileOnions;
+
 static std::string embeddedDir="/t/cryt/shadow";
 
 /*convert lex of insert into string*/
@@ -39,7 +43,6 @@ convert_insert(const LEX &lex)
 
 /************************************************************************************************/
 
-
 /*encrypt one onion to get item. if the onion exists, then return directly*/
 static Item *
 my_encrypt_item_layers(const Item &i, onion o, const OnionMeta &om,
@@ -69,8 +72,15 @@ my_typical_rewrite_insert_type(const Item &i, const FieldMeta &fm,
     for (auto it : fm.orderedOnionMetas()) {
         const onion o = it.first->getValue();
         OnionMeta * const om = it.second;      
+        if(o==oAGG){
+            std::vector<std::string> &tempFileVector = fileOnions[o];
+            std::string in(tempFileVector.back());
+            tempFileVector.pop_back();
+            l->push_back(MySQLFieldTypeToItem(static_cast<enum_field_types>(253),in));
+        }else{
             l->push_back(my_encrypt_item_layers(i, o, *om, a, IV));
         }
+    }
     if (fm.getHasSalt()) {
         l->push_back(new Item_int(static_cast<ulonglong>(salt)));
     }
@@ -149,7 +159,7 @@ static std::string getInsertResults(Analysis a,LEX* lex){
 }
 
 
-static void testInsertHandler(std::string query){
+static void testInsertHandler(std::string query,std::string db){
     std::unique_ptr<Connect> e_conn(Connect::getEmbedded(embeddedDir));
     std::unique_ptr<SchemaInfo> schema(new SchemaInfo());
     std::function<DBMeta *(DBMeta *const)> loadChildren =
@@ -164,36 +174,62 @@ static void testInsertHandler(std::string query){
     loadChildren(schema.get()); 
     const std::unique_ptr<AES_KEY> &TK = std::unique_ptr<AES_KEY>(getKey(std::string("113341234")));
     //just like what we do in Rewrite::rewrite,dispatchOnLex
-    Analysis analysis(std::string("tdb"),*schema,TK,
+    Analysis analysis(std::string(db),*schema,TK,
                         SECURITY_RATING::SENSITIVE);
     std::unique_ptr<query_parse> p;
     p = std::unique_ptr<query_parse>(
-                new query_parse("tdb", query));
+                new query_parse(db, query));
     LEX *const lex = p->lex();
 
     std::cout<<getInsertResults(analysis,lex)<<std::endl;
 }
 
+//fileOnions
+static 
+void initFileOnions(std::string filename,std::string dataPrefix="allTables/table_MGVGBABDMU/"){
+    std::ifstream infile(filename);
+    std::string line;
+    
+    std::string onionAnnoName;
+    while(getline(infile,line)){
+        int index = line.find(":");
+        std::string head = line.substr(0,index);
+        std::string next = line.substr(index+1);
+        onionAnnoName = next;
+    }
+
+    std::vector<std::string> res;
+    load_string_file(dataPrefix+onionAnnoName,res,256);
+    std::reverse(res.begin(),res.end());
+    fileOnions[oAGG] = std::move(res);
+}
+
 int
-main() {
+main(int argc,char**argv) {
     char *buffer;
     if((buffer = getcwd(NULL, 0)) == NULL){
         perror("getcwd error");
     }
     embeddedDir = std::string(buffer)+"/shadow";
+
+    if(argc!=2){
+        assert(0);        
+    }
+    std::string db(argv[1]);
+
+    initFileOnions("conf/already.cnf");
+
     const std::string master_key = "113341234";
     ConnectionInfo ci("localhost", "root", "letmein",3306);
     SharedProxyState *shared_ps = 
                      new SharedProxyState(ci, embeddedDir , master_key, determineSecurityRating());
     assert(shared_ps!=NULL);
-    std::string query1 = "insert into child values(1,\"ZHAOYUN\")";
+    std::string query1 = "insert into child values(1,\"ZHAOYUN\"),(2,'XC'),(3,'KK')";
     std::vector<std::string> querys{query1};
-
     for(auto item:querys){
         std::cout<<item<<std::endl;
-        testInsertHandler(item);
+        testInsertHandler(item,db);
         std::cout<<std::endl;
     }
-
     return 0;
 }

debug/fast_insert_connect.cc → debug/fast_insert.cc.back

@@ -11,12 +11,81 @@
 #include <main/ddl_handler.hh>
 #include <main/CryptoHandlers.hh>
 #include <main/rewrite_main.hh>
-#include "wrapper/insert_lib.hh"
-
 static std::string embeddedDir="/t/cryt/shadow";
 
+/*convert lex of insert into string*/
+static
+std::ostream&
+simple_insert(std::ostream &out, LEX &lex){
+        String s;
+        THD *t = current_thd;
+        const char* cmd = "INSERT";
+        out<<cmd<<" ";
+        lex.select_lex.table_list.first->print(t, &s, QT_ORDINARY);
+        out << "INTO " << s;
+        out << " values " << noparen(lex.many_values);
+        return out;
+}
+
+static
+std::string
+convert_insert(const LEX &lex)
+{
+    std::ostringstream o;
+    simple_insert(o,const_cast<LEX &>(lex));
+    return o.str();
+}
+
+
+/************************************************************************************************/
+
+
+/*encrypt one onion to get item. if the onion exists, then return directly*/
+static Item *
+my_encrypt_item_layers(const Item &i, onion o, const OnionMeta &om,
+                    const Analysis &a, uint64_t IV) {
+    assert(!RiboldMYSQL::is_null(i));
+    const auto &enc_layers = a.getEncLayers(om);
+    assert_s(enc_layers.size() > 0, "onion must have at least one layer");
+    const Item *enc = &i;
+    Item *new_enc = NULL;
+    for (const auto &it : enc_layers) {
+        new_enc = it->encrypt(*enc, IV);
+        assert(new_enc);
+        enc = new_enc;
+    }
+    assert(new_enc && new_enc != &i);
+    return new_enc;
+}
 
 
+/*encrypt the item to get a set of onions, in the parater l,salt is also added*/
+static
+void
+my_typical_rewrite_insert_type(const Item &i, const FieldMeta &fm,
+                            Analysis &a, std::vector<Item *> *l) {
+    const uint64_t salt = fm.getHasSalt() ? randomValue() : 0;
+    uint64_t IV = salt;
+    for (auto it : fm.orderedOnionMetas()) {
+        const onion o = it.first->getValue();
+        OnionMeta * const om = it.second;
+        l->push_back(my_encrypt_item_layers(i, o, *om, a, IV));
+    }
+    if (fm.getHasSalt()) {
+        l->push_back(new Item_int(static_cast<ulonglong>(salt)));
+    }
+}
+
+static
+void myRewriteInsertHelper(const Item &i, const FieldMeta &fm, Analysis &a,
+                           List<Item> *const append_list){
+    std::vector<Item *> l;
+    my_typical_rewrite_insert_type(i,fm,a,&l);
+    for (auto it : l) {
+        append_list->push_back(it);
+    }
+}
+
 static std::string getInsertResults(Analysis a,LEX* lex){
         LEX *const new_lex = copyWithTHD(lex);
         const std::string &table =
@@ -49,7 +118,10 @@ static std::string getInsertResults(Analysis a,LEX* lex){
                 }
                 List<Item> *const newList0 = new List<Item>();
                 if (li->elements != fmVec.size()) {
-                    assert(0);
+                    TEST_TextMessageError(0 == li->elements
+                                         && NULL == lex->field_list.head(),
+                                          "size mismatch between fields"
+                                          " and values!");
                 } else {
                     auto it0 = List_iterator<Item>(*li);
                     auto fmVecIt = fmVec.begin();
@@ -60,6 +132,8 @@ static std::string getInsertResults(Analysis a,LEX* lex){
                         if (!i) {
                             break;
                         }
+                        //fetch values, and use fieldMeta to facilitate rewrite
+                        //every filed should be encrypted with onions of encryption
                         myRewriteInsertHelper(*i, **fmVecIt, a, newList0);
                         ++fmVecIt;
                     }
@@ -71,7 +145,6 @@ static std::string getInsertResults(Analysis a,LEX* lex){
             }
             new_lex->many_values = newList;
         }
-
         return convert_insert(*new_lex);
 }
 
@@ -121,5 +194,6 @@ main() {
         testInsertHandler(item);
         std::cout<<std::endl;
     }
+
     return 0;
 }

wrapper/reuse.cc

@@ -443,6 +443,7 @@ void load_num_file(std::string filename,std::vector<std::string> &res){
 void load_string_file(std::string filename, std::vector<std::string> &res,unsigned long length){
     char *buf = new char[length];
     int fd = open(filename.c_str(),O_RDONLY);
+    if(fd==-1) assert(0);//reading from -1 may cause errors
     while(read(fd,buf,length)!=0){
         res.push_back(std::string(buf,length));
     }