add modified version of escape_string

debug/final_load.cc

@@ -201,7 +201,7 @@ static ResType load_files(std::string db, std::string table){
 
 static
 void local_wrapper(const Item &i, const FieldMeta &fm, Analysis &a,
-                           List<Item> *const append_list){
+                           List<Item> *const append_list) {
     //append_list->push_back(&(const_cast<Item&>(i)));
     //do not use the plain strategy 
     std::vector<Item *> l;
@@ -280,7 +280,7 @@ main(int argc, char* argv[]){
     const std::string head = std::string("INSERT INTO `")+db+"`.`"+annoTableName+"` ";
 
     /*reencryption to get the encrypted insert!!!*/
-    for(auto &row:res.rows){
+    for(auto &row:res.rows) {
         List<List_item> newList;
         List<Item> *const newList0 = new List<Item>();
         for(auto i=0u;i<res.names.size();i++){

debug/final_store.cc

@@ -64,11 +64,24 @@ static void write_meta(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> &
 }
 
 static
-void write_raw_data_to_files(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> &res ,string db,string table){
+void write_raw_data_to_files(rawMySQLReturnValue& resraw,
+                             std::vector<FieldMetaTrans> &res,
+                             string db,
+                             string table,
+                             std::vector<int> vlen,
+                             std::vector<std::string> vstr,
+                             std::string vname) {
     //write metafiles
     write_meta(resraw,res,db,table);
     //write datafiles
     write_row_data(resraw,db,table);
+
+    //write special swap
+    std::string prefix = std::string("data/")+db+"/"+table+"/";
+    std::string swpfile = prefix+vname;
+    std::string swpkeyfile = prefix+vname+"key";
+    (void)swpfile;
+    (void)swpkeyfile;
 }
 
 static void store(std::string db, std::string table){
@@ -89,8 +102,15 @@ static void store(std::string db, std::string table){
 
     //generate the backup query and then fetch the tuples
     std::string backup_query = getTestQuery(*schema,res,db,table);
-
-    rawMySQLReturnValue resraw =  executeAndGetResultRemote(globalConn,backup_query);
+    
+    std::vector<int> vlen;
+    std::vector<std::string> vstr;
+    std::string vname;
+    rawMySQLReturnValue resraw =  executeAndGetResultRemoteWithOneVariableLen(globalConn,
+                                                                              backup_query,
+                                                                              vlen,
+                                                                              vstr,
+                                                                              vname);
 
     //then we should set the type and length of FieldMetaTrans
     auto types = resraw.fieldTypes;
@@ -113,7 +133,7 @@ static void store(std::string db, std::string table){
     }
 
     //write the tuples into files
-    write_raw_data_to_files(resraw,res,db,table);
+    write_raw_data_to_files(resraw,res,db,table,vlen,vstr,vname);
 }
 
 int

debug/final_store.cc.back

+#include "wrapper/common.hh"
+#include "wrapper/reuse.hh"
+static std::string embeddedDir="/t/cryt/shadow";
+//global map, for each client, we have one WrapperState which contains ProxyState.
+static std::map<std::string, WrapperState*> clients;
+//This connection mimics the behaviour of MySQL-Proxy
+Connect  *globalConn;
+
+static void init(std::string ip,int port){
+    std::string client="192.168.1.1:1234";
+    //one Wrapper per user.
+    clients[client] = new WrapperState();    
+    //Connect phase
+    ConnectionInfo ci("localhost", "root", "letmein",port);
+    const std::string master_key = "113341234";
+    char *buffer;
+    if((buffer = getcwd(NULL, 0)) == NULL){  
+        perror("getcwd error");  
+    }
+    embeddedDir = std::string(buffer)+"/shadow";
+    SharedProxyState *shared_ps = 
+			new SharedProxyState(ci, embeddedDir , master_key, 
+                                            determineSecurityRating());
+    assert(0 == mysql_thread_init());
+    //we init embedded database here.
+    clients[client]->ps = std::unique_ptr<ProxyState>(new ProxyState(*shared_ps));
+    clients[client]->ps->safeCreateEmbeddedTHD();
+    //Connect end!!
+    globalConn = new Connect(ip, ci.user, ci.passwd, port);
+}
+
+
+//query for testing purposes
+static
+std::string getTestQuery(SchemaInfo &schema, std::vector<FieldMetaTrans> &tfds,
+                                     std::string db="tdb",std::string table="student1"){
+    std::string res = "SELECT ";
+    const std::unique_ptr<IdentityMetaKey> dbmeta_key(new IdentityMetaKey(db));
+    //get databaseMeta, search in the map
+    DatabaseMeta * dbm = schema.getChild(*dbmeta_key);
+    const TableMeta & tbm = *((*dbm).getChild(IdentityMetaKey(table)));
+    std::string annotablename = tbm.getAnonTableName();
+
+    //then a list of onion names
+    for(auto tf:tfds){
+        for(auto item : tf.getChoosenOnionName()){
+            res += item;
+            res += " , ";
+        }
+        //actually the salt should be selected if RND is used,this should be changed later.
+    	if(tf.getHasSalt()){
+            res += tf.getSaltName() + " , ";
+        }
+    }
+    res = res.substr(0,res.size()-2);
+    res = res + "FROM `"+db+std::string("`.`")+annotablename+"`";
+    return res;
+}
+
+static void write_meta(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> &res,string db,string table){
+    TableMetaTrans mf(db,table,res);
+    mf.set_db_table(db,table);
+    mf.serialize();
+}
+
+static
+void write_raw_data_to_files(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> &res ,string db,string table){
+    //write metafiles
+    write_meta(resraw,res,db,table);
+    //write datafiles
+    write_row_data(resraw,db,table);
+}
+
+static void store(std::string db, std::string table){
+    std::unique_ptr<SchemaInfo> schema =  myLoadSchemaInfo(embeddedDir);
+    //get all the fields in the tables
+    std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
+
+    //transform the field so that selected onions can be used
+    std::vector<FieldMetaTrans> res;
+    for(auto i=0u;i<fms.size();i++){
+        FieldMetaTrans ft;
+        res.push_back(ft);
+        res.back().trans(fms[i]);
+    }
+
+    /*this is our strategy, each field should be able to choose the selected onion*/
+    storeStrategies(res);
+
+    //generate the backup query and then fetch the tuples
+    std::string backup_query = getTestQuery(*schema,res,db,table);
+
+    rawMySQLReturnValue resraw =  executeAndGetResultRemote(globalConn,backup_query);
+
+    //then we should set the type and length of FieldMetaTrans
+    auto types = resraw.fieldTypes;
+    auto lengths = resraw.lengths;
+    int base_types = 0;
+    int base_lengths = 0;
+    for(auto &item:res){
+        vector<int> tempTypes;
+        vector<int> tempLengths;
+        for(unsigned int i=0u;i<item.getChoosenOnionName().size();i++){
+            tempTypes.push_back(types[base_types++]);
+            tempLengths.push_back(lengths[base_lengths++]);
+        }
+        item.setChoosenFieldTypes(tempTypes);
+        item.setChoosenFieldLengths(tempLengths);
+        if(item.getHasSalt()){//also this should be changed.
+            item.setSaltType(types[base_types++]);
+            item.setSaltLength(lengths[base_lengths++]);
+        }
+    }
+
+    //write the tuples into files
+    write_raw_data_to_files(resraw,res,db,table);
+}
+
+int
+main(int argc, char* argv[]){    
+    std::string db="tdb",table="student";
+    std::string ip="127.0.0.1";
+    int port=3306;
+    if(argc==4){
+        ip = std::string(argv[1]);
+        db = std::string(argv[2]);
+        table = std::string(argv[3]);
+    }
+    init(ip,port);
+    store(db,table);
+    return 0;
+}

debug/micro_insert_str.cc

@@ -12,7 +12,10 @@ main(int argc,char ** argv) {
 
     std::vector<string> create{
         "use micro_db;",
-        "insert into str_table values('a'),('b'),('c'),('d'),('e');"
+//        "insert into str_table values('aaaaaaaa'),('bbbbbbbbaaa'),('ccccccccaaaa'),('ddddddddddddd'),('eeeeeeeedddddddddd');"
+        "insert into str_table values('ddddddddddddd'),('eeeeeeeedddddddddd');"
+    
+
     };
     big_proxy b("tdb",ip,"root","letmein",3306);
     for(auto item:create){

debug/test_escape.cc

+#include "wrapper/common.hh"
+#include "wrapper/reuse.hh"
+#include "util/util.hh"
+static std::string embeddedDir="/t/cryt/shadow";
+//global map, for each client, we have one WrapperState which contains ProxyState.
+static std::map<std::string, WrapperState*> clients;
+//This connection mimics the behaviour of MySQL-Proxy
+Connect  *globalConn;
+
+static void init(std::string ip,int port){
+    std::string client="192.168.1.1:1234";
+    //one Wrapper per user.
+    clients[client] = new WrapperState();    
+    //Connect phase
+    ConnectionInfo ci("localhost", "root", "letmein",port);
+    const std::string master_key = "113341234";
+    char *buffer;
+    if((buffer = getcwd(NULL, 0)) == NULL){  
+        perror("getcwd error");  
+    }
+    embeddedDir = std::string(buffer)+"/shadow";
+    SharedProxyState *shared_ps = 
+			new SharedProxyState(ci, embeddedDir , master_key, 
+                                            determineSecurityRating());
+    assert(0 == mysql_thread_init());
+    //we init embedded database here.
+    clients[client]->ps = std::unique_ptr<ProxyState>(new ProxyState(*shared_ps));
+    clients[client]->ps->safeCreateEmbeddedTHD();
+    //Connect end!!
+    globalConn = new Connect(ip, ci.user, ci.passwd, port);
+}
+
+
+int
+main(int argc, char* argv[]){    
+    std::string db="tdb",table="student";
+    std::string ip="127.0.0.1";
+    int port=3306;
+    if(argc==2){
+        ip = std::string(argv[1]);
+    }
+    init(ip,port);
+//    std::string es("a\0\nb\\n",6);
+//   std::cout<<es<<std::endl;
+//    std::unique_ptr<Connect> con(globalConn);
+//    std::string res = escapeString(con,es);
+//    char * buf = new char[100];
+//    char * buf2 = new char[100];
+//    escape_string_for_mysql_modify(buf,es.c_str(),es.size());
+//    printf("%s\n",buf);
+//    reverse_escape_string_for_mysql_modify(buf2,buf);
+//    printf("%s",buf2);
+
+    char * original = new char[1024];
+    for(int i=0;i<1023;i++){
+        original[i]=i;
+    }
+    original[1023]='\0';
+    std::cout<<strlen(original)<<std::endl;
+    std::string oriStr(original,1023);
+    std::cout<<oriStr<<std::endl;
+    std::cout<<strlen(original)<<std::endl;
+
+    std::unique_ptr<Connect> con(globalConn);
+    std::string esp_cryptdb = escapeString(con,oriStr);
+    
+    //reverse the string escaped by cryptdb.
+    char* reverse = new char[2048];
+    size_t len = reverse_escape_string_for_mysql_modify(reverse,esp_cryptdb.c_str());
+    std::string back(reverse,len);
+    assert(back==oriStr);
+
+    char* esp_modify = new char[2048];
+    len = escape_string_for_mysql_modify(esp_modify,original,1023);
+    std::string esp_my(esp_modify,len);
+    assert(esp_my==esp_cryptdb);
+
+    return 0;
+}
+
+

debug/test_field_name_and_length.cc

+/*
+To make this work properly, you should at least make sure that the database tdb exists.
+*/
+#include <iostream>
+#include <vector>
+#include <functional>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <main/Connect.hh>
+#include <main/rewrite_util.hh>
+#include <main/sql_handler.hh>
+#include <main/dml_handler.hh>
+#include <main/ddl_handler.hh>
+#include <main/CryptoHandlers.hh>
+#include "wrapper/reuse.hh"
+
+using std::vector;
+using std::string;
+
+struct batch{
+    vector<vector<string>> rows;
+    vector<string> fields;
+    vector<enum_field_types> types;
+    vector<vector<int>> lengths;
+};
+
+
+static
+batch executeAndGetBatch(Connect* connect, std::string query){
+    MYSQL *conn = connect->get_conn();
+    batch bt;
+    if (mysql_query(conn,query.c_str())) {
+        assert(0);
+    }
+    MYSQL_RES *result = mysql_store_result(conn);
+    if (result == NULL) {
+        return bt;
+    }
+    
+    int num_fields = mysql_num_fields(result);
+    if(num_fields==0) return bt;
+
+    std::vector<std::vector<std::string>> rows;
+    std::vector<std::string> fields;
+    std::vector<enum_field_types> types;
+    std::vector<std::vector<int>> lengths;
+
+    MYSQL_FIELD *field;
+    for(int i=0;i<num_fields;i++){
+        field = mysql_fetch_field(result);
+        if(field!=NULL){
+            fields.push_back(field->name);
+            types.push_back(field->type);
+        }else{
+            std::cout<<"field error"<<std::endl;
+        }
+    }
+
+    MYSQL_ROW row;
+    while ((row = mysql_fetch_row(result))) {
+        unsigned long * fieldLen = mysql_fetch_lengths(result);
+        vector<string> curRow;
+        vector<int> curLength;
+        for(int i = 0; i < num_fields; i++) {
+            if(row[i]==NULL){
+                curRow.push_back(string("NULL"));
+                curLength.push_back(0);
+            }else{
+                curRow.push_back(string(row[i],fieldLen[i]));
+                curLength.push_back(fieldLen[i]);
+            }
+        }
+        rows.push_back(curRow);
+        lengths.push_back(curLength);
+    }
+    bt.rows = rows;
+    bt.fields = fields;
+    bt.lengths = lengths;
+    bt.types = types;
+    return bt;
+}
+
+static
+void getTables(Connect*connect, std::string db,std::string table){
+    std::string query = std::string("SELECT * FROM ")+table+";";
+    batch bt = executeAndGetBatch(connect,query);
+    (void)bt;
+    return ;
+}
+
+int
+main(int argc,char** argv) {
+    if(argc!=3){
+        assert(0);
+    }
+    std::string db(argv[1]);
+    std::string table(argv[2]);
+
+    const std::string master_key = "113341234";
+    ConnectionInfo ci("localhost", "root", "letmein",3306);
+    Connect *connect = new Connect(ci.server, ci.user, ci.passwd, ci.port);
+    connect->execute(string("use ")+db);
+    getTables(connect,db,table);
+    return 0;
+}

debug/test_memory.cc

@@ -27,6 +27,7 @@ void initGfb(vector<string> field_names,
              vector<int>field_lengths,
                                      std::string db,std::string table){
     std::map<std::string,std::vector<std::string>> gfm;
+
     std::string prefix = std::string("data/")+db+"/"+table+"/";
     for(unsigned int i=0u; i<field_names.size(); i++) {
         std::string filename = prefix + field_names[i];
@@ -37,12 +38,17 @@ void initGfb(vector<string> field_names,
             load_string_file(filename,column,field_lengths[i]);
         }
         gfm[field_names[i]] = std::move(column);
+
 //        vector<string>().swap(gfm[field_names[i]]);
+
     }//get memory 31%
 
     for(unsigned int i=0u; i<field_names.size(); i++) {
         vector<string>().swap(gfm[field_names[i]]);
+        //gfm.erase(gfm[field_names[i]]);
     }
+
+
     return;
 }
 

load_ip.sh

+./mtl/final_load_low_memory localhost micro_db int_table

store_ip.sh

+./mtl/final_store localhost micro_db int_table

util/util.cc

@@ -580,3 +580,97 @@ test64bitZZConversions()
 
     return true;
 }
+
+
+
+
+size_t 
+escape_string_for_mysql_modify(char *to,
+                               const char *from, size_t length) {
+  const char *to_start= to;
+  const char *endD, *to_end=to_start + 2*length;
+  bool overflow= false;
+  for (endD= from + length; from < endD; from++){
+    char escape= 0;
+    switch (*from) {
+    case 0:                             /* Must be escaped for 'mysql' */
+      escape= '0';
+      break;
+    case '\n':                          /* Must be escaped for logs */
+      escape= 'n';
+      break;
+    case '\r':
+      escape= 'r';
+      break;
+    case '\\':
+      escape= '\\';
+      break;
+    case '\'':
+      escape= '\'';
+      break;
+    case '"':                           /* Better safe than sorry */
+      escape= '"';
+      break;
+    case '\032':                        /* This gives problems on Win32 */
+      escape= 'Z';
+      break;
+    }
+    if (escape)
+    {
+      if (to + 2 > to_end)
+      {
+        overflow= true;
+        break;
+      }
+      *to++= '\\';
+      *to++= escape;
+    }
+    else
+    {
+      if (to + 1 > to_end)
+      {
+        overflow= true;
+        break;
+      }
+      *to++= *from;
+    }
+  }
+  *to= 0;
+  return overflow ? (size_t) -1 : (size_t) (to - to_start);
+}
+
+
+size_t 
+reverse_escape_string_for_mysql_modify(char *to,
+                               const char *from) {
+    char * to_start = to;
+    size_t length = strlen(from);
+    for(size_t i=0u;i<length;){
+        if(from[i]=='\\') {
+            i++;
+            if(from[i]=='0'){
+                *to='\0';
+            }else if(from[i]=='n'){
+                *to='\n';
+            }else if(from[i]=='r'){
+                *to='\r';
+            }else if(from[i]=='\\'){
+                *to='\\';
+            }else if(from[i]=='\''){
+                *to='\'';
+            }else if(from[i]=='"'){
+                *to='"';
+            }else if(from[i]=='Z'){
+                 *to='\032';
+            }
+
+            i++;
+        }else{
+            *to=from[i];
+            i++;
+        }
+        to++;
+    }
+    *to='\0';
+    return to - to_start;
+}

util/util.hh

@@ -573,3 +573,13 @@ destructThenFree(void *const p)
 }
 
 bool test64bitZZConversions();
+
+
+size_t 
+reverse_escape_string_for_mysql_modify(char *to,
+                               const char *from);
+
+
+size_t 
+escape_string_for_mysql_modify(char *to,
+                               const char *from, size_t length);

wrapper/reuse.cc

@@ -359,6 +359,56 @@ executeAndGetResultRemote(Connect * curConn,std::string query){
     return myRaw;
 }
 
+
+rawMySQLReturnValue 
+executeAndGetResultRemoteWithOneVariableLen(Connect * curConn,
+                                           std::string query,
+                                           std::vector<int> &vlen,
+                                           std::vector<std::string> &vstr,
+                                           std::string &vname) {
+    std::unique_ptr<DBResult> dbres;
+    curConn->execute(query, &dbres);
+    rawMySQLReturnValue myRaw;
+    
+    if(dbres==nullptr||dbres->n==NULL){
+        std::cout<<"no results"<<std::endl;
+        return myRaw;
+    }
+
+    int num = mysql_num_rows(dbres->n);
+    
+    int numOfFields = mysql_num_fields(dbres->n);
+
+    MYSQL_FIELD *field;
+    MYSQL_ROW row;
+
+    if(num!=0){
+        while( (row = mysql_fetch_row(dbres->n)) ){
+            //what's the difference between fieldlen
+	    unsigned long * fieldLen = mysql_fetch_lengths(dbres->n);
+            std::vector<std::string> curRow;
+            for(int i=0;i<numOfFields;i++){
+                if (i == 0) {
+                    while( (field = mysql_fetch_field(dbres->n)) ) {
+                        myRaw.fieldNames.push_back(std::string(field->name));
+                        myRaw.fieldTypes.push_back(field->type);
+                        //myRaw.lengths.push_back(field->length);
+                        //myRaw.lengths.push_back(fieldLen[i]);
+                        myRaw.lengths.push_back(field->max_length);
+                        myRaw.maxlengths.push_back(field->max_length);
+                        //cout<<field->length<<"::"<<field->max_length<<endl;
+                    }
+                }
+                if(row[i]==NULL) curRow.push_back("NULL");
+                else curRow.push_back(std::string(row[i],fieldLen[i]));
+            }
+            myRaw.rowValues.push_back(curRow);
+        }
+    }
+    return myRaw;
+}
+
+
 void
 write_row_data(rawMySQLReturnValue& resraw,std::string db,std::string table,std::string prefix){
     std::vector<FILE*> data_files;

wrapper/reuse.hh

@@ -149,6 +149,14 @@ rawMySQLReturnValue
 executeAndGetResultRemote(Connect * curConn,std::string query);
 
 
+
+rawMySQLReturnValue 
+executeAndGetResultRemoteWithOneVariableLen(Connect * curConn,
+                                           std::string query,
+                                           std::vector<int> &vlen,
+                                           std::vector<std::string> &vstr,
+std::string &vname);
+
 void
 write_row_data(rawMySQLReturnValue& resraw,std::string db,std::string table,std::string prefix="data/");