able to execute and get column data directly

debug/final_store.cc

@@ -64,24 +64,22 @@ static void write_meta(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> &
 }
 
 static
-void write_raw_data_to_files(rawMySQLReturnValue& resraw,
-                             std::vector<FieldMetaTrans> &res,
-                             string db,
-                             string table,
-                             std::vector<int> vlen,
-                             std::vector<std::string> vstr,
-                             std::string vname) {
+void write_raw_data_to_files(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> &res ,string db,string table){
     //write metafiles
     write_meta(resraw,res,db,table);
     //write datafiles
     write_row_data(resraw,db,table);
+    std::string prefix = std::string("data/") +db+"/"+table+"/";
+    std::vector<std::string> filenames;
+    for(auto item:resraw.fieldNames){
+        item=prefix+item;
+        filenames.push_back(item);
+    }
+
+
+
+
 
-    //write special swap
-    std::string prefix = std::string("data/")+db+"/"+table+"/";
-    std::string swpfile = prefix+vname;
-    std::string swpkeyfile = prefix+vname+"key";
-    (void)swpfile;
-    (void)swpkeyfile;
 }
 
 static void store(std::string db, std::string table){
@@ -102,15 +100,8 @@ static void store(std::string db, std::string table){
 
     //generate the backup query and then fetch the tuples
     std::string backup_query = getTestQuery(*schema,res,db,table);
-    
-    std::vector<int> vlen;
-    std::vector<std::string> vstr;
-    std::string vname;
-    rawMySQLReturnValue resraw =  executeAndGetResultRemoteWithOneVariableLen(globalConn,
-                                                                              backup_query,
-                                                                              vlen,
-                                                                              vstr,
-                                                                              vname);
+
+    rawMySQLReturnValue resraw =  executeAndGetResultRemote(globalConn,backup_query);
 
     //then we should set the type and length of FieldMetaTrans
     auto types = resraw.fieldTypes;
@@ -133,7 +124,7 @@ static void store(std::string db, std::string table){
     }
 
     //write the tuples into files
-    write_raw_data_to_files(resraw,res,db,table,vlen,vstr,vname);
+    write_raw_data_to_files(resraw,res,db,table);
 }
 
 int

debug/test_connect.cc

+/*1. store data as column files, and restore data as plaintext insert query
+* 2. plaintext insert query should be able to recover directly
+* 3. should be able to used exsisting data to reduce the computation overhead(to be implemented)
+*/
+#include <stdlib.h>
+#include <stdio.h>
+#include <string>
+#include <map>
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <vector>
+#include <algorithm>
+#include "wrapper/reuse.hh"
+#include "wrapper/common.hh"
+#include "wrapper/insert_lib.hh"
+#include "util/constants.hh"
+using std::cout;
+using std::cin;
+using std::endl;
+using std::vector;
+using std::string;
+using std::to_string;
+static std::string embeddedDir="/t/cryt/shadow";
+char * globalEsp=NULL;
+int num_of_pipe = 4;
+//global map, for each client, we have one WrapperState which contains ProxyState.
+static std::map<std::string, WrapperState*> clients;
+//This connection mimics the behaviour of MySQL-Proxy
+Connect  *globalConn;
+/*for each field, convert the format to FieldMeta_Wrapper*/
+static void init(){
+    std::string client="192.168.1.1:1234";
+    //one Wrapper per user.
+    clients[client] = new WrapperState();    
+    //Connect phase
+    ConnectionInfo ci("localhost", "root", "letmein",3306);
+    const std::string master_key = "113341234";
+    char *buffer;
+    if((buffer = getcwd(NULL, 0)) == NULL){  
+        perror("getcwd error");  
+    }
+    embeddedDir = std::string(buffer)+"/shadow";
+    SharedProxyState *shared_ps = 
+			new SharedProxyState(ci, embeddedDir , master_key, 
+                                            determineSecurityRating());
+    assert(0 == mysql_thread_init());
+    //we init embedded database here.
+    clients[client]->ps = std::unique_ptr<ProxyState>(new ProxyState(*shared_ps));
+    clients[client]->ps->safeCreateEmbeddedTHD();
+    //Connect end!!
+    globalConn = new Connect(ci.server, ci.user, ci.passwd, ci.port);
+}
+
+//========================================================================================//
+
+
+
+int
+main(int argc, char* argv[]){
+    init();
+    create_embedded_thd(0);
+    std::string db="tdb",table="student";
+    std::string ip="localhost";
+    if(argc==4){
+        ip = std::string(argv[1]);
+        db = std::string(argv[2]);
+        table = std::string(argv[3]);
+    }
+
+    std::string query = "select * from ttt.student;";
+    executeAndGetColumnData(globalConn,query);
+    return 0;
+}
+

wrapper/reuse.cc

@@ -361,6 +361,47 @@ executeAndGetResultRemote(Connect * curConn,std::string query){
 }
 
 
+MySQLColumnData
+executeAndGetColumnData(Connect * curConn,std::string query){
+    std::unique_ptr<DBResult> dbres;
+    curConn->execute(query, &dbres);
+    MySQLColumnData myRaw;
+    
+    if(dbres==nullptr||dbres->n==NULL){
+        std::cout<<"no results"<<std::endl;
+        return myRaw;
+    }
+
+    int num = mysql_num_rows(dbres->n);
+    int numOfFields = mysql_num_fields(dbres->n);
+
+    MYSQL_FIELD *field;
+    MYSQL_ROW row;
+
+    while( (field = mysql_fetch_field(dbres->n)) ) {
+        myRaw.fieldNames.push_back(std::string(field->name));
+        myRaw.fieldTypes.push_back(field->type);
+        myRaw.maxLengths.push_back(field->max_length);
+    }
+
+    for(int i=0;i<numOfFields;i++){
+        myRaw.columnData.push_back(std::vector<std::string>());
+    }
+
+    if(num!=0){
+        while( (row = mysql_fetch_row(dbres->n)) ){
+            //what's the difference between fieldlen
+	    unsigned long * fieldLen = mysql_fetch_lengths(dbres->n);
+            for(int i=0;i<numOfFields;i++){
+                if(row[i]==NULL) myRaw.columnData[i].push_back("NULL");
+                else myRaw.columnData[i].push_back(std::string(row[i],fieldLen[i]));
+            }
+        }
+    }
+    return myRaw;
+}
+
+
 rawMySQLReturnValue 
 executeAndGetResultRemoteWithOneVariableLen(Connect * curConn,
                                            std::string query,
@@ -437,8 +478,6 @@ write_row_data(rawMySQLReturnValue& resraw,std::string db,std::string table,std:
    string should be escaped before being written into the file */
 void
 writeRowdataEscapeString(const std::vector<std::string> &column,
-                      std::string db,
-                      std::string table,
                       std::string columnFilename,
                       unsigned int maxLength) {
     FILE* dataFileHandler = fopen(columnFilename.c_str(),"w");
@@ -458,8 +497,6 @@ writeRowdataEscapeString(const std::vector<std::string> &column,
 */
 void 
 writeRowdataNum(const std::vector<std::string> &column,
-                      std::string db,
-                      std::string table,
                       std::string columnFilename) {
     FILE* dataFileHandler = fopen(columnFilename.c_str(),"w");
     const std::string token = "\n";
@@ -471,6 +508,63 @@ writeRowdataNum(const std::vector<std::string> &column,
 }
 
 
+void loadFileEscape(std::string filename,
+                    std::vector<std::string> &res,
+                    unsigned int maxLength) {
+    std::ifstream infile(filename);
+    std::string line;
+    char *buf = new char[2*maxLength+1u];
+    while(std::getline(infile,line)){
+        size_t len = reverse_escape_string_for_mysql_modify(buf,line.c_str());
+        std::string temp(buf,len);
+        res.push_back(temp);
+    }
+    infile.close();
+}
+
+void loadFileEscapeLimitCount(std::string filename,
+                    std::vector<std::string> &res,
+                    unsigned int maxLength,int limit) {
+    std::ifstream infile(filename);
+    std::string line;
+    char *buf = new char[2*maxLength+1u];
+    int localCount=0;
+    while(std::getline(infile,line)){
+        size_t len = reverse_escape_string_for_mysql_modify(buf,line.c_str());
+        std::string temp(buf,len);
+        res.push_back(temp);
+        localCount++;
+        if(localCount==limit) break;
+    }
+    infile.close();
+}
+
+
+void 
+loadFileNoEscape(std::string filename,            
+              std::vector<std::string> &res) {
+    std::ifstream infile(filename);                  
+    std::string line;
+    while(std::getline(infile,line)) {
+        res.push_back(line);
+    }
+}
+
+void 
+loadFileNoEscapeLimitCount(std::string filename,
+                 std::vector<std::string> &res,int limit){
+    std::ifstream infile(filename);                  
+    std::string line;
+    int localCount=0;
+    while(std::getline(infile,line)) {
+        res.push_back(line);
+        localCount++;
+        if(localCount==limit) break;
+    }
+}
+
+
+
 
 
 STORE_STRATEGY currentStrategy = STORE_STRATEGY::ALL;
@@ -529,17 +623,6 @@ void load_num_file(std::string filename,std::vector<std::string> &res){
     infile.close();
 }
 
-void load_file_escape(std::string filename,
-                      std::vector<std::string> &res) {
-    std::ifstream infile(filename);
-    std::string line;
-    while(std::getline(infile,line)){
-        res.push_back(std::move(line));
-    }
-    infile.close();
-}
-
-
 void 
 load_num_file_count(std::string filename,
               std::vector<std::string> &res,
@@ -613,6 +696,3 @@ getStringItem(std::string s){
 }
 
 
-
-
-

wrapper/reuse.hh

@@ -40,6 +40,17 @@ struct rawMySQLReturnValue {
     void show();
 };
 
+/*Raw return value from mysql*/
+struct MySQLColumnData {
+    std::vector<std::vector<std::string>> columnData;/*data tuples*/
+    std::vector<std::string> fieldNames;
+    std::vector<enum_field_types> fieldTypes;
+    std::vector<int> maxLengths;/*what's the difference between length and maxlength?*/
+};
+
+
+
+
 //representation of one field.
 struct FieldMeta_Wrapper{
     bool hasSalt;
@@ -148,6 +159,9 @@ void transform_to_rawMySQLReturnValue(rawMySQLReturnValue & str,ResType & item);
 rawMySQLReturnValue
 executeAndGetResultRemote(Connect * curConn,std::string query);
 
+MySQLColumnData
+executeAndGetColumnData(Connect * conn,std::string query);
+
 
 
 rawMySQLReturnValue 
@@ -206,21 +220,35 @@ load_string_file_count(std::string filename,
                        int count);
 
 
+void 
+loadFileEscape(std::string filename,
+                    std::vector<std::string> &res,
+                    unsigned int maxLength);
 
-void load_file_escape(std::string filename,
-                      std::vector<std::string> &res);
+void 
+loadFileNoEscape(std::string filename,            
+              std::vector<std::string> &res);
 
 
 void
 writeRowdataEscapeString(const std::vector<std::string> &column,
-                      std::string db,
-                      std::string table,
                       std::string columnFilename,
                       unsigned int maxLength);
 
 
 void 
 writeRowdataNum(const std::vector<std::string> &column,
-                      std::string db,
-                      std::string table,
                       std::string columnFilename);
+
+
+void loadFileEscapeLimitCount(std::string filename,
+                    std::vector<std::string> &res,
+                    unsigned int maxLength,int limit);
+
+void 
+loadFileNoEscapeLimitCount(std::string filename,
+                 std::vector<std::string> &res,int limit);
+
+
+//Connect * initEmbeddedAndRemoteConnection(std::string ip,int port);
+