Skip to content

P
Practical-Cryptdb
Commit 51ff0337 authored by yiwenshao's avatar yiwenshao
Browse files
Options

able to execute and get column data directly

parent 90fd537c
Hide whitespace changes
Inline Side-by-side
Showing with 221 additions and 345 deletions
debug/final_load.cc deleted 100644 → 0
View file @ 90fd537c
/*1. store data as column files, and restore data as plaintext insert query
* 2. plaintext insert query should be able to recover directly
* 3. should be able to used exsisting data to reduce the computation overhead(to be implemented)
*/
#include <stdlib.h>
#include <stdio.h>
#include <string>
#include <map>
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
#include <algorithm>
#include "wrapper/reuse.hh"
#include "wrapper/common.hh"
#include "wrapper/insert_lib.hh"
#include "util/constants.hh"
using std::cout;
using std::cin;
using std::endl;
using std::vector;
using std::string;
using std::to_string;
static std::string embeddedDir="/t/cryt/shadow";
char * globalEsp=NULL;
int num_of_pipe = 4;
//global map, for each client, we have one WrapperState which contains ProxyState.
static std::map<std::string, WrapperState*> clients;
//This connection mimics the behaviour of MySQL-Proxy
Connect *globalConn;
/*for each field, convert the format to FieldMeta_Wrapper*/
static void init(){
std::string client="192.168.1.1:1234";
//one Wrapper per user.
clients[client] = new WrapperState();
//Connect phase
ConnectionInfo ci("localhost", "root", "letmein",3306);
const std::string master_key = "113341234";
char *buffer;
if((buffer = getcwd(NULL, 0)) == NULL){
perror("getcwd error");
}
embeddedDir = std::string(buffer)+"/shadow";
SharedProxyState *shared_ps =
new SharedProxyState(ci, embeddedDir , master_key,
determineSecurityRating());
assert(0 == mysql_thread_init());
//we init embedded database here.
clients[client]->ps = std::unique_ptr<ProxyState>(new ProxyState(*shared_ps));
clients[client]->ps->safeCreateEmbeddedTHD();
//Connect end!!
globalConn = new Connect(ci.server, ci.user, ci.passwd, ci.port);
}
//========================================================================================//
fullBackUp gfb;
struct batch{
vector<string> field_names;
vector<int> field_types;
vector<int> field_lengths;
};
batch ggbt;
/*should choose the right decryption onion*/
static
std::shared_ptr<ReturnMeta> getReturnMeta(std::vector<FieldMeta*> fms,
std::vector<FieldMetaTrans> &tfds){
assert(fms.size()==tfds.size());
std::shared_ptr<ReturnMeta> myReturnMeta = std::make_shared<ReturnMeta>();
int pos=0;
//construct OLK
for(auto i=0u;i<tfds.size();i++){
//the order is DET,OPE,ASHE,AGG. other onions are not decryptable!!
int index = getDecryptionOnionIndex(tfds[i]);
if(index==-1) assert(0);
onion o = tfds[i].getChoosenOnionO()[index];
SECLEVEL l = tfds[i].getOriginalFieldMeta()->getOnionMeta(o)->getSecLevel();
FieldMeta *k = tfds[i].getOriginalFieldMeta();
OLK curOLK(o,l,k);
bool use_salt = false;
if(needsSalt(curOLK))
use_salt = true;
addToReturn(myReturnMeta.get(),pos++,curOLK,use_salt,k->getFieldName());
if(use_salt)
addSaltToReturn(myReturnMeta.get(),pos++);
ggbt.field_types.push_back(tfds[i].getChoosenFieldTypes()[index]);
ggbt.field_names.push_back(tfds[i].getChoosenOnionName()[index]);
ggbt.field_lengths.push_back(tfds[i].getChoosenFieldLengths()[index]);
if(use_salt){
ggbt.field_types.push_back(tfds[i].getSaltType());
ggbt.field_names.push_back(tfds[i].getSaltName());
ggbt.field_lengths.push_back(tfds[i].getSaltLength());
}
}
return myReturnMeta;
}
/*init global full backup. */
static
void initGfb(std::vector<FieldMetaTrans> &res,std::string db,std::string table){
vector<string> field_names;
vector<int> field_types;
vector<int> field_lengths;
/*choosen onions should all be included in gfb. salt is also included
it's hard to decide whether a FieldMetaTrans has salt because the senmantic is different from that of FieldMeta.
*/
for(auto &item:res){
for(auto i:item.getChoosenOnionName()){
field_names.push_back(i);
}
for(auto i:item.getChoosenFieldTypes()){
field_types.push_back(i);
}
for(auto i:item.getChoosenFieldLengths()){
field_lengths.push_back(i);
}
if(item.getHasSalt()){
field_names.push_back(item.getSaltName());
field_types.push_back(item.getSaltType());
field_lengths.push_back(item.getSaltLength());
}
}
gfb.field_names = field_names;
gfb.field_types = field_types;
gfb.field_lengths = field_lengths;
//then we should read the vector
std::string prefix = std::string("data/")+db+"/"+table+"/";
for(unsigned int i=0u; i<gfb.field_names.size(); i++) {
std::string filename = prefix + gfb.field_names[i];
std::vector<std::string> column;
if(IS_NUM(gfb.field_types[i])){
load_num_file_count(filename,column,constGlobalConstants.loadCount);
}else{
load_string_file_count(filename,column,gfb.field_lengths[i],constGlobalConstants.loadCount);
}
gfb.annoOnionNameToFileVector[gfb.field_names[i]] = std::move(column);
}
//init another map
for(unsigned int i=0;i<gfb.field_names.size();i++){
gfb.annoOnionNameToType[gfb.field_names[i]] = gfb.field_types[i];
}
}
/*load file, decrypt, and then return data plain fields in the type ResType*/
static ResType load_files(std::string db, std::string table){
std::unique_ptr<SchemaInfo> schema = myLoadSchemaInfo(embeddedDir);
//get all the fields in the tables.
std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
TableMetaTrans res_meta = loadTableMetaTrans(db,table);
std::vector<FieldMetaTrans> res = res_meta.getFts();
for(unsigned int i=0;i<fms.size();i++){
res[i].trans(fms[i]);
}
//then we should load all the fields available
initGfb(res,db,table);
std::shared_ptr<ReturnMeta> rm = getReturnMeta(fms,res);
vector<string> field_names = ggbt.field_names;
vector<int> field_types = ggbt.field_types;
vector<int> field_lengths = ggbt.field_lengths;
//why do we need this??
create_embedded_thd(0);
rawMySQLReturnValue resraw;
// vector<vector<string>> resss_field = loadTableFieldsForDecryption(db,
// table,field_names, field_types, field_lengths);
vector<vector<string>> res_field;
for(auto item:field_names){
res_field.push_back(gfb.annoOnionNameToFileVector[item]);
}
//then transform it to ress_fields
unsigned int length = res_field[0].size();
vector<vector<string>> ress_field;
for(unsigned int i=0u;i<length;i++){
vector<string> row;
for(unsigned int j=0u;j<res_field.size();j++){
row.push_back(res_field[j][i]);
}
ress_field.push_back(row);
}
resraw.rowValues = ress_field;
resraw.fieldNames = field_names;
for(unsigned int i=0;i<field_types.size();++i){
resraw.fieldTypes.push_back(static_cast<enum_field_types>(field_types[i]));
}
ResType rawtorestype = rawMySQLReturnValue_to_ResType(false, &resraw);
auto finalresults = decryptResults(rawtorestype,*rm);
return finalresults;
}
static
void local_wrapper(const Item &i, const FieldMeta &fm, Analysis &a,
List<Item> *const append_list) {
//append_list->push_back(&(const_cast<Item&>(i)));
//do not use the plain strategy
std::vector<Item *> l;
const uint64_t salt = fm.getHasSalt() ? randomValue() : 0;
uint64_t IV = salt;
for (auto it : fm.orderedOnionMetas()) {
const onion o = it.first->getValue();
OnionMeta * const om = it.second;
std::string annoOnionName = om->getAnonOnionName();
if(gfb.annoOnionNameToFileVector.find(annoOnionName)!=gfb.annoOnionNameToFileVector.end()){
enum_field_types type = static_cast<enum_field_types>(gfb.annoOnionNameToType[annoOnionName]);
std::vector<std::string> &tempFileVector = gfb.annoOnionNameToFileVector[annoOnionName];
std::string in = tempFileVector.back();
if(IS_NUM(type)){
//std::string in("11");
unsigned int len = annoOnionName.size();
if(len>4u&&annoOnionName.substr(len-4)=="ASHE"){
l.push_back(MySQLFieldTypeToItem(type,in));
}else{
l.push_back( new (current_thd->mem_root)
Item_int(static_cast<ulonglong>(valFromStr(in))) );
}
}else{
//std::string in("hehe");
l.push_back(MySQLFieldTypeToItem(type,in));
}
tempFileVector.pop_back();
//l.push_back(&(const_cast<Item&>(i)));
}else{
l.push_back(my_encrypt_item_layers(i, o, *om, a, IV));
}
}
std::string saltName = fm.getSaltName();
if (fm.getHasSalt()) {
if(gfb.annoOnionNameToFileVector.find(saltName)!=gfb.annoOnionNameToFileVector.end()){
std::vector<std::string> &tempFileVector = gfb.annoOnionNameToFileVector[saltName];
std::string in = tempFileVector.back();
// enum_field_types type = static_cast<enum_field_types>(gfb.annoOnionNameToType[saltName]);
// l.push_back(MySQLFieldTypeToItem(type,in));
l.push_back( new (current_thd->mem_root)
Item_int(static_cast<ulonglong>(valFromStr(in)))
);
tempFileVector.pop_back();
}else{
l.push_back(new Item_int(static_cast<ulonglong>(salt)));
}
}
for (auto it : l) {
append_list->push_back(it);
}
}
int
main(int argc, char* argv[]){
init();
create_embedded_thd(0);
std::string db="tdb",table="student";
std::string ip="localhost";
if(argc==4){
ip = std::string(argv[1]);
db = std::string(argv[2]);
table = std::string(argv[3]);
}
std::unique_ptr<SchemaInfo> schema = myLoadSchemaInfo(embeddedDir);
schema.get();
const std::unique_ptr<AES_KEY> &TK = std::unique_ptr<AES_KEY>(getKey(std::string("113341234")));
Analysis analysis(db, *schema, TK, SECURITY_RATING::SENSITIVE);
/*choose decryption onion, load and decrypt to plain text*/
ResType res = load_files(db,table);
std::string annoTableName = analysis.getTableMeta(db,table).getAnonTableName();
const std::string head = std::string("INSERT INTO `")+db+"`.`"+annoTableName+"` ";
/*reencryption to get the encrypted insert!!!*/
for(auto &row:res.rows) {
List<List_item> newList;
List<Item> *const newList0 = new List<Item>();
for(auto i=0u;i<res.names.size();i++){
std::string field_name = res.names[i];
FieldMeta & fm = analysis.getFieldMeta(db,table,field_name);
local_wrapper(*row[i],fm,analysis,newList0);
}
newList.push_back(newList0);
std::ostringstream o;
insertManyValues(o,newList);
std::cout<<(head+o.str())<<std::endl;
}
return 0;
}
debug/final_store.cc
View file @ 51ff0337
...@@ -64,24 +64,22 @@ static void write_meta(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> & ...@@ -64,24 +64,22 @@ static void write_meta(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> &
} }
static static
void write_raw_data_to_files(rawMySQLReturnValue& resraw, void write_raw_data_to_files(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> &res ,string db,string table){
std::vector<FieldMetaTrans> &res,
string db,
string table,
std::vector<int> vlen,
std::vector<std::string> vstr,
std::string vname) {
//write metafiles //write metafiles
write_meta(resraw,res,db,table); write_meta(resraw,res,db,table);
//write datafiles //write datafiles
write_row_data(resraw,db,table); write_row_data(resraw,db,table);
std::string prefix = std::string("data/") +db+"/"+table+"/";
std::vector<std::string> filenames;
for(auto item:resraw.fieldNames){
item=prefix+item;
filenames.push_back(item);
}
//write special swap
std::string prefix = std::string("data/")+db+"/"+table+"/";
std::string swpfile = prefix+vname;
std::string swpkeyfile = prefix+vname+"key";
(void)swpfile;
(void)swpkeyfile;
} }
static void store(std::string db, std::string table){ static void store(std::string db, std::string table){
...@@ -102,15 +100,8 @@ static void store(std::string db, std::string table){ ...@@ -102,15 +100,8 @@ static void store(std::string db, std::string table){
//generate the backup query and then fetch the tuples //generate the backup query and then fetch the tuples
std::string backup_query = getTestQuery(*schema,res,db,table); std::string backup_query = getTestQuery(*schema,res,db,table);
std::vector<int> vlen; rawMySQLReturnValue resraw = executeAndGetResultRemote(globalConn,backup_query);
std::vector<std::string> vstr;
std::string vname;
rawMySQLReturnValue resraw = executeAndGetResultRemoteWithOneVariableLen(globalConn,
backup_query,
vlen,
vstr,
vname);
//then we should set the type and length of FieldMetaTrans //then we should set the type and length of FieldMetaTrans
auto types = resraw.fieldTypes; auto types = resraw.fieldTypes;
...@@ -133,7 +124,7 @@ static void store(std::string db, std::string table){ ...@@ -133,7 +124,7 @@ static void store(std::string db, std::string table){
} }
//write the tuples into files //write the tuples into files
write_raw_data_to_files(resraw,res,db,table,vlen,vstr,vname); write_raw_data_to_files(resraw,res,db,table);
} }
int int
......
debug/test_connect.cc 0 → 100644
View file @ 51ff0337
/*1. store data as column files, and restore data as plaintext insert query
* 2. plaintext insert query should be able to recover directly
* 3. should be able to used exsisting data to reduce the computation overhead(to be implemented)
*/
#include <stdlib.h>
#include <stdio.h>
#include <string>
#include <map>
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
#include <algorithm>
#include "wrapper/reuse.hh"
#include "wrapper/common.hh"
#include "wrapper/insert_lib.hh"
#include "util/constants.hh"
using std::cout;
using std::cin;
using std::endl;
using std::vector;
using std::string;
using std::to_string;
static std::string embeddedDir="/t/cryt/shadow";
char * globalEsp=NULL;
int num_of_pipe = 4;
//global map, for each client, we have one WrapperState which contains ProxyState.
static std::map<std::string, WrapperState*> clients;
//This connection mimics the behaviour of MySQL-Proxy
Connect *globalConn;
/*for each field, convert the format to FieldMeta_Wrapper*/
static void init(){
std::string client="192.168.1.1:1234";
//one Wrapper per user.
clients[client] = new WrapperState();
//Connect phase
ConnectionInfo ci("localhost", "root", "letmein",3306);
const std::string master_key = "113341234";
char *buffer;
if((buffer = getcwd(NULL, 0)) == NULL){
perror("getcwd error");
}
embeddedDir = std::string(buffer)+"/shadow";
SharedProxyState *shared_ps =
new SharedProxyState(ci, embeddedDir , master_key,
determineSecurityRating());
assert(0 == mysql_thread_init());
//we init embedded database here.
clients[client]->ps = std::unique_ptr<ProxyState>(new ProxyState(*shared_ps));
clients[client]->ps->safeCreateEmbeddedTHD();
//Connect end!!
globalConn = new Connect(ci.server, ci.user, ci.passwd, ci.port);
}
//========================================================================================//
int
main(int argc, char* argv[]){
init();
create_embedded_thd(0);
std::string db="tdb",table="student";
std::string ip="localhost";
if(argc==4){
ip = std::string(argv[1]);
db = std::string(argv[2]);
table = std::string(argv[3]);
}
std::string query = "select * from ttt.student;";
executeAndGetColumnData(globalConn,query);
return 0;
}
wrapper/reuse.cc
View file @ 51ff0337
...@@ -361,6 +361,47 @@ executeAndGetResultRemote(Connect * curConn,std::string query){ ...@@ -361,6 +361,47 @@ executeAndGetResultRemote(Connect * curConn,std::string query){
} }
MySQLColumnData
executeAndGetColumnData(Connect * curConn,std::string query){
std::unique_ptr<DBResult> dbres;
curConn->execute(query, &dbres);
MySQLColumnData myRaw;
if(dbres==nullptr||dbres->n==NULL){
std::cout<<"no results"<<std::endl;
return myRaw;
}
int num = mysql_num_rows(dbres->n);
int numOfFields = mysql_num_fields(dbres->n);
MYSQL_FIELD *field;
MYSQL_ROW row;
while( (field = mysql_fetch_field(dbres->n)) ) {
myRaw.fieldNames.push_back(std::string(field->name));
myRaw.fieldTypes.push_back(field->type);
myRaw.maxLengths.push_back(field->max_length);
}
for(int i=0;i<numOfFields;i++){
myRaw.columnData.push_back(std::vector<std::string>());
}
if(num!=0){
while( (row = mysql_fetch_row(dbres->n)) ){
//what's the difference between fieldlen
unsigned long * fieldLen = mysql_fetch_lengths(dbres->n);
for(int i=0;i<numOfFields;i++){
if(row[i]==NULL) myRaw.columnData[i].push_back("NULL");
else myRaw.columnData[i].push_back(std::string(row[i],fieldLen[i]));
}
}
}
return myRaw;
}
rawMySQLReturnValue rawMySQLReturnValue
executeAndGetResultRemoteWithOneVariableLen(Connect * curConn, executeAndGetResultRemoteWithOneVariableLen(Connect * curConn,
std::string query, std::string query,
...@@ -437,8 +478,6 @@ write_row_data(rawMySQLReturnValue& resraw,std::string db,std::string table,std: ...@@ -437,8 +478,6 @@ write_row_data(rawMySQLReturnValue& resraw,std::string db,std::string table,std:
string should be escaped before being written into the file */ string should be escaped before being written into the file */
void void
writeRowdataEscapeString(const std::vector<std::string> &column, writeRowdataEscapeString(const std::vector<std::string> &column,
std::string db,
std::string table,
std::string columnFilename, std::string columnFilename,
unsigned int maxLength) { unsigned int maxLength) {
FILE* dataFileHandler = fopen(columnFilename.c_str(),"w"); FILE* dataFileHandler = fopen(columnFilename.c_str(),"w");
...@@ -458,8 +497,6 @@ writeRowdataEscapeString(const std::vector<std::string> &column, ...@@ -458,8 +497,6 @@ writeRowdataEscapeString(const std::vector<std::string> &column,
*/ */
void void
writeRowdataNum(const std::vector<std::string> &column, writeRowdataNum(const std::vector<std::string> &column,
std::string db,
std::string table,
std::string columnFilename) { std::string columnFilename) {
FILE* dataFileHandler = fopen(columnFilename.c_str(),"w"); FILE* dataFileHandler = fopen(columnFilename.c_str(),"w");
const std::string token = "\n"; const std::string token = "\n";
...@@ -471,6 +508,63 @@ writeRowdataNum(const std::vector<std::string> &column, ...@@ -471,6 +508,63 @@ writeRowdataNum(const std::vector<std::string> &column,
} }
void loadFileEscape(std::string filename,
std::vector<std::string> &res,
unsigned int maxLength) {
std::ifstream infile(filename);
std::string line;
char *buf = new char[2*maxLength+1u];
while(std::getline(infile,line)){
size_t len = reverse_escape_string_for_mysql_modify(buf,line.c_str());
std::string temp(buf,len);
res.push_back(temp);
}
infile.close();
}
void loadFileEscapeLimitCount(std::string filename,
std::vector<std::string> &res,
unsigned int maxLength,int limit) {
std::ifstream infile(filename);
std::string line;
char *buf = new char[2*maxLength+1u];
int localCount=0;
while(std::getline(infile,line)){
size_t len = reverse_escape_string_for_mysql_modify(buf,line.c_str());
std::string temp(buf,len);
res.push_back(temp);
localCount++;
if(localCount==limit) break;
}
infile.close();
}
void
loadFileNoEscape(std::string filename,
std::vector<std::string> &res) {
std::ifstream infile(filename);
std::string line;
while(std::getline(infile,line)) {
res.push_back(line);
}
}
void
loadFileNoEscapeLimitCount(std::string filename,
std::vector<std::string> &res,int limit){
std::ifstream infile(filename);
std::string line;
int localCount=0;
while(std::getline(infile,line)) {
res.push_back(line);
localCount++;
if(localCount==limit) break;
}
}
STORE_STRATEGY currentStrategy = STORE_STRATEGY::ALL; STORE_STRATEGY currentStrategy = STORE_STRATEGY::ALL;
...@@ -529,17 +623,6 @@ void load_num_file(std::string filename,std::vector<std::string> &res){ ...@@ -529,17 +623,6 @@ void load_num_file(std::string filename,std::vector<std::string> &res){
infile.close(); infile.close();
} }
void load_file_escape(std::string filename,
std::vector<std::string> &res) {
std::ifstream infile(filename);
std::string line;
while(std::getline(infile,line)){
res.push_back(std::move(line));
}
infile.close();
}
void void
load_num_file_count(std::string filename, load_num_file_count(std::string filename,
std::vector<std::string> &res, std::vector<std::string> &res,
...@@ -613,6 +696,3 @@ getStringItem(std::string s){ ...@@ -613,6 +696,3 @@ getStringItem(std::string s){
} }
wrapper/reuse.hh
View file @ 51ff0337
...@@ -40,6 +40,17 @@ struct rawMySQLReturnValue { ...@@ -40,6 +40,17 @@ struct rawMySQLReturnValue {
void show(); void show();
}; };
/*Raw return value from mysql*/
struct MySQLColumnData {
std::vector<std::vector<std::string>> columnData;/*data tuples*/
std::vector<std::string> fieldNames;
std::vector<enum_field_types> fieldTypes;
std::vector<int> maxLengths;/*what's the difference between length and maxlength?*/
};
//representation of one field. //representation of one field.
struct FieldMeta_Wrapper{ struct FieldMeta_Wrapper{
bool hasSalt; bool hasSalt;
...@@ -148,6 +159,9 @@ void transform_to_rawMySQLReturnValue(rawMySQLReturnValue & str,ResType & item); ...@@ -148,6 +159,9 @@ void transform_to_rawMySQLReturnValue(rawMySQLReturnValue & str,ResType & item);
rawMySQLReturnValue rawMySQLReturnValue
executeAndGetResultRemote(Connect * curConn,std::string query); executeAndGetResultRemote(Connect * curConn,std::string query);
MySQLColumnData
executeAndGetColumnData(Connect * conn,std::string query);
rawMySQLReturnValue rawMySQLReturnValue
...@@ -206,21 +220,35 @@ load_string_file_count(std::string filename, ...@@ -206,21 +220,35 @@ load_string_file_count(std::string filename,
int count); int count);
void
loadFileEscape(std::string filename,
std::vector<std::string> &res,
unsigned int maxLength);
void load_file_escape(std::string filename, void
std::vector<std::string> &res); loadFileNoEscape(std::string filename,
std::vector<std::string> &res);
void void
writeRowdataEscapeString(const std::vector<std::string> &column, writeRowdataEscapeString(const std::vector<std::string> &column,
std::string db,
std::string table,
std::string columnFilename, std::string columnFilename,
unsigned int maxLength); unsigned int maxLength);
void void
writeRowdataNum(const std::vector<std::string> &column, writeRowdataNum(const std::vector<std::string> &column,
std::string db,
std::string table,
std::string columnFilename); std::string columnFilename);
void loadFileEscapeLimitCount(std::string filename,
std::vector<std::string> &res,
unsigned int maxLength,int limit);
void
loadFileNoEscapeLimitCount(std::string filename,
std::vector<std::string> &res,int limit);
//Connect * initEmbeddedAndRemoteConnection(std::string ip,int port);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment