Skip to content

P
Practical-Cryptdb
Commit 251a19a3 authored by yiwenshao's avatar yiwenshao
Browse files
Options

able to use onionindex in load and store

parent 114f77df
Hide whitespace changes
Inline Side-by-side
Showing with 175 additions and 339 deletions
debug/common.hh
View file @ 251a19a3
#pragma once
/*class metadata_file{
class metadata_files{
public:
string db,table;
int num_of_fields;
vector<string> field_types;
vector<int> field_lengths;
vector<string> field_names;
vector<int> choosen_onions;
/*selected fields*/
vector<vector<int>> selected_field_types;
vector<vector<int>> selected_field_lengths;
vector<vector<string>> selected_field_names;
vector<string> has_salt;
vector<int> dec_onion_index;/*should be 0,1,2,3...*/
std::string serialize_vec_int(std::string s,vector<int> vec_int);
std::string serialize_vec_str(std::string s,vector<string> vec_str);
vector<string> string_to_vec_str(string line);
vector<int> string_to_vec_int(string line);
static bool make_path(std::string directory);
public:
void set_db(std::string idb){db=idb;}
std::string get_db(){return db;}
void set_table(std::string itable){table=itable;}
std::string get_table(){return table;}
void set_db_table(std::string idb,std::string itable){db=idb;table=itable;}
void set_num_of_fields(int num){num_of_fields = num;}
int get_num_of_fields(){return num_of_fields;}
void set_field_types(vector<string> input){field_types = input;}
std::vector<std::string> & get_field_types(){return field_types;}
void set_field_lengths(vector<int> input){field_lengths = input;}
std::vector<int> & get_field_lengths(){return field_lengths;}
void set_field_names(vector<string> input){field_names = input;}
std::vector<std::string> & get_field_names(){return field_names;}
void set_choosen_onions(vector<int> input){choosen_onions = input;}
std::vector<int>& get_choosen_onions(){return choosen_onions;}
void serilize();
void deserialize(std::string filename);
void show();
static bool make_path(std::string directory){
struct stat st;
if(directory.size()==0||directory[0]=='/') return false;
if(directory.back()=='/') directory.pop_back();
int start = 0,next=0;
while(stat(directory.c_str(),&st)==-1&&next!=-1){
next = directory.find('/',start);
if(next!=-1){
string sub = directory.substr(0,next);
if(stat(sub.c_str(),&st)==-1)
mkdir(sub.c_str(),0700);
start = next + 1;
}else{
mkdir(directory.c_str(),0700);
}
}
return true;
}
};
void metadata_file::serilize(){
FILE * localmeta = NULL;
string prefix = string("data/")+db+"/"+table;
make_path(prefix);
localmeta = fopen((prefix+"/metadata.data").c_str(),"w");
string s = string("database:")+db;
s+="\n";
fwrite(s.c_str(),1,s.size(),localmeta);
s = string("table:")+table;
s+="\n";
fwrite(s.c_str(),1,s.size(),localmeta);
void set_selected_field_types(vector<vector<int>> input){selected_field_types = input;}
vector<vector<int>> &get_selected_field_types(){return selected_field_types;};
s = string("num_of_fields:")+to_string(num_of_fields)+"\n";
fwrite(s.c_str(),1,s.size(),localmeta);
void set_selected_field_lengths(vector<vector<int>> input){selected_field_lengths = input;}
vector<vector<int>> &get_selected_field_lengths(){return selected_field_lengths;}
s = string("field_types:");
for(auto item:field_types){
s+=item+=" ";
}
s.back()='\n';
fwrite(s.c_str(),1,s.size(),localmeta);
void set_selected_field_names(vector<vector<string>> input){selected_field_names = input;}
vector<vector<string>> &get_selected_field_names(){return selected_field_names;}
s = string("field_lengths:");
for(auto item : field_lengths){
s+=to_string(item)+=" ";
}
s.back()='\n';
fwrite(s.c_str(),1,s.size(),localmeta);
void set_dec_onion_index(vector<int> input){dec_onion_index = input;}
vector<int> &get_dec_onion_index(){return dec_onion_index;}
s = string("field_names:");
for(auto item : field_names){
s+=item+=" ";
}
s.back()='\n';
fwrite(s.c_str(),1,s.size(),localmeta);
void set_has_salt(vector<string> input){has_salt = input;}
vector<string> &get_has_salt(){return has_salt;}
s = string("choosen_onions:");
for(auto item : choosen_onions){
void serialize();
void deserialize(std::string filename);
};
std::string metadata_files::serialize_vec_int(std::string s,vector<int> vec_int){
s+=":";
for(auto item:vec_int){
s+=to_string(item)+=" ";
}
s.back()='\n';
fwrite(s.c_str(),1,s.size(),localmeta);
fclose(localmeta);
return s;
}
void metadata_file::deserialize(std::string filename){
filename = string("data/")+db+"/"+table+"/"+filename;
std::ifstream infile(filename);
string line;
while(std::getline(infile,line)){
int index = line.find(":");
string head = line.substr(0,index);
if(head=="database"){
set_db(line.substr(index+1));
}else if(head=="table"){
set_table(line.substr(index+1));
}else if(head=="num_of_fields"){
set_num_of_fields(std::stoi(line.substr(index+1)));
}else if(head=="field_types"){
string types = line.substr(index+1);
int start=0,next=0;
std::vector<std::string> tmp;
while((next=types.find(' ',start))!=-1){
string item = types.substr(start,next-start);
tmp.push_back(item);
start = next+1;
}
string item = types.substr(start);
tmp.push_back(item);
set_field_types(tmp);
}else if(head=="field_lengths"){
string lengths = line.substr(index+1);
int start=0,next=0;
std::vector<int> tmp;
while((next=lengths.find(' ',start))!=-1){
string item = lengths.substr(start,next-start);
tmp.push_back(std::stoi(item));
start = next+1;
}
string item = lengths.substr(start);
tmp.push_back(std::stoi(item));
set_field_lengths(tmp);
}else if(head=="field_names"){
std::vector<std::string> tmp;
string names = line.substr(index+1);
int start=0,next=0;
while((next=names.find(' ',start))!=-1){
string item = names.substr(start,next-start);
tmp.push_back(item);
start = next+1;
}
string item = names.substr(start);
tmp.push_back(item);
set_field_names(tmp);
}else if(head=="choosen_onions"){
std::vector<int> tmp;
string c_onions = line.substr(index+1);
int start=0,next=0;
while((next=c_onions.find(' ',start))!=-1){
string item = c_onions.substr(start,next-start);
tmp.push_back(std::stoi(item));
start = next+1;
}
string item = c_onions.substr(start);
tmp.push_back(std::stoi(item));
set_choosen_onions(tmp);
}
}
infile.close();
}
void metadata_file::show(){
cout<<db<<endl;
cout<<table<<endl;
cout<<num_of_fields<<endl;
for(auto item:field_types){
cout<<item<<"\t";
}
cout<<endl;
for(auto item:field_lengths){
cout<<item<<"\t";
}
cout<<endl;
for(auto item:field_names){
cout<<item<<"\t";
}
cout<<endl;
for(auto item:choosen_onions){
cout<<item<<"\t";
std::string metadata_files::serialize_vec_str(std::string s,vector<string> vec_str){
s+=":";
for(auto item:vec_str){
s+=item+=" ";
}
cout<<endl;
s.back()='\n';
return s;
}
*/
/*******************************************************************************************************************/
class metadata_files{
public:
string db,table;
/*selected fields*/
vector<vector<int>> selected_field_types;
vector<vector<int>> selected_field_lengths;
vector<vector<string>> selected_field_names;
vector<string> has_salt;
vector<int> dec_onion_index;/*should be 0,1,2,3...*/
std::string serialize_vec_int(std::string s,vector<int> vec_int){
s+=":";
for(auto item:vec_int){
s+=to_string(item)+=" ";
}
s.back()='\n';
return s;
}
std::string serialize_vec_str(std::string s,vector<string> vec_str){
s+=":";
for(auto item:vec_str){
s+=item+=" ";
}
s.back()='\n';
return s;
}
vector<string> string_to_vec_str(string line){
int start=0,next=0;
std::vector<std::string> tmp;
while((next=line.find(' ',start))!=-1){
string item = line.substr(start,next-start);
tmp.push_back(item);
start = next+1;
}
string item = line.substr(start);
vector<string> metadata_files::string_to_vec_str(string line){
int start=0,next=0;
std::vector<std::string> tmp;
while((next=line.find(' ',start))!=-1){
string item = line.substr(start,next-start);
tmp.push_back(item);
return tmp;
start = next+1;
}
string item = line.substr(start);
tmp.push_back(item);
return tmp;
}
vector<int> string_to_vec_int(string line){
int start=0,next=0;
std::vector<int> tmp;
while((next=line.find(' ',start))!=-1){
string item = line.substr(start,next-start);
tmp.push_back(std::stoi(item));
start = next+1;
}
string item = line.substr(start);
vector<int> metadata_files::string_to_vec_int(string line){
int start=0,next=0;
std::vector<int> tmp;
while((next=line.find(' ',start))!=-1){
string item = line.substr(start,next-start);
tmp.push_back(std::stoi(item));
return tmp;
start = next+1;
}
public:
void set_db(std::string idb){db=idb;}
std::string get_db(){return db;}
void set_table(std::string itable){table=itable;}
std::string get_table(){return table;}
void set_db_table(std::string idb,std::string itable){db=idb;table=itable;}
void set_selected_field_types(vector<vector<int>> input){selected_field_types = input;}
void set_selected_field_lengths(vector<vector<int>> input){selected_field_lengths = input;}
void set_selected_field_names(vector<vector<string>> input){selected_field_names = input;}
void set_dec_onion_index(vector<int> input){dec_onion_index = input;}
void set_has_salt(vector<string> input){has_salt = input;}
void serialize();
void deserialize(std::string filename);
string item = line.substr(start);
tmp.push_back(std::stoi(item));
return tmp;
}
static bool make_path(std::string directory){
struct stat st;
if(directory.size()==0||directory[0]=='/') return false;
if(directory.back()=='/') directory.pop_back();
int start = 0,next=0;
while(stat(directory.c_str(),&st)==-1&&next!=-1){
next = directory.find('/',start);
if(next!=-1){
string sub = directory.substr(0,next);
if(stat(sub.c_str(),&st)==-1)
mkdir(sub.c_str(),0700);
start = next + 1;
}else{
mkdir(directory.c_str(),0700);
}
bool metadata_files::make_path(std::string directory){
struct stat st;
if(directory.size()==0||directory[0]=='/') return false;
if(directory.back()=='/') directory.pop_back();
int start = 0,next=0;
while(stat(directory.c_str(),&st)==-1&&next!=-1){
next = directory.find('/',start);
if(next!=-1){
string sub = directory.substr(0,next);
if(stat(sub.c_str(),&st)==-1)
mkdir(sub.c_str(),0700);
start = next + 1;
}else{
mkdir(directory.c_str(),0700);
}
return true;
}
};
return true;
}
void metadata_files::serialize(){
FILE * localmeta = NULL;
......@@ -316,9 +159,9 @@ void metadata_files::deserialize(std::string filename){
string head = line.substr(0,index);
if(head == "INDEX"){
//INIT HERE
selected_field_types.push_back(vector<int>());
selected_field_lengths.push_back(vector<int>());
selected_field_names.push_back(vector<string>());
// selected_field_types.push_back(vector<int>());
// selected_field_lengths.push_back(vector<int>());
// selected_field_names.push_back(vector<string>());
}else if(head=="database"){
set_db(line.substr(index+1));
}else if(head=="table"){
......
debug/load.cc
View file @ 251a19a3
......@@ -54,18 +54,62 @@ vector<T> flat_vec(vector<vector<T>> &input){
}
return res;
}
struct batch{
vector<string> field_names;
vector<int> field_types;
vector<int> field_lengths;
};
static vector<vector<string>> load_table_fields(metadata_files & input) {
static
batch get_batch(metadata_files & input,std::vector<transField> &tfms){
vector<vector<int>> selected_field_types = input.get_selected_field_types();
vector<vector<int>> selected_field_lengths = input.get_selected_field_lengths();
vector<vector<string>> selected_field_names = input.get_selected_field_names();
vector<int> dec_onion_index = input.get_dec_onion_index();
vector<string> has_salt = input.get_has_salt();
vector<string> field_names;
vector<int> field_types;
vector<int> field_lengths;
for(auto i=0u;i<tfms.size();i++){
int index = dec_onion_index[i];
string dec_field_name = tfms[i].fields[index];
auto f = find(selected_field_names[i].begin(),selected_field_names[i].end(),dec_field_name);
assert(f!=selected_field_names[i].end());
int j = f - selected_field_names[i].begin();
if(has_salt[i]==string("true")){
field_names.push_back(selected_field_names[i][j]);
field_types.push_back(selected_field_types[i][j]);
field_lengths.push_back(selected_field_lengths[i][j]);
field_names.push_back(selected_field_names[i].back());
field_types.push_back(selected_field_types[i].back());
field_lengths.push_back(selected_field_lengths[i].back());
}else{
assert(1==2);
}
}
batch bt;
bt.field_names = field_names;
bt.field_types = field_types;
bt.field_lengths = field_lengths;
return bt;
}
#include <algorithm>
static vector<vector<string>> load_table_fields(metadata_files & input,std::vector<transField> &tfms) {
string db = input.get_db();
string table = input.get_table();
vector<vector<string>> res;
string prefix = string("data/")+db+"/"+table+"/";
vector<string> datafiles;
auto field_names = flat_vec(input.selected_field_names);
auto field_types = flat_vec(input.selected_field_types);
auto field_lengths = flat_vec(input.selected_field_lengths);
auto bt = get_batch(input,tfms);
vector<string> field_names = bt.field_names;
vector<int> field_types = bt.field_types;
vector<int> field_lengths = bt.field_lengths;
vector<string> datafiles;
for(auto item:field_names){
datafiles.push_back(prefix+item);
}
......@@ -87,37 +131,32 @@ static vector<vector<string>> load_table_fields(metadata_files & input) {
return res;
}
static ResType load_files(std::string db="tdb", std::string table="student"){
std::unique_ptr<SchemaInfo> schema = myLoadSchemaInfo();
//get all the fields in the tables.
std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
auto res = getTransField(fms);
std::vector<enum_field_types> types;//Added
for(auto item:fms){
types.push_back(item->getSqlType());
}//Add new field form FieldMeta
if(types.size()==1){
//to be
}
metadata_files res_meta = load_meta(db,table);
std::shared_ptr<ReturnMeta> rm = getReturnMeta(fms,res);
//why do we need this??
create_embedded_thd(0);
rawMySQLReturnValue resraw;
//load fields in the stored file
vector<vector<string>> res_field = load_table_fields(res_meta);
vector<vector<string>> res_field = load_table_fields(res_meta,res);
resraw.rowValues = res_field;
auto field_names = flat_vec(res_meta.selected_field_names);
auto field_types = flat_vec(res_meta.selected_field_types);
auto field_lengths = flat_vec(res_meta.selected_field_lengths);
// auto field_names = flat_vec(res_meta.selected_field_names);
// auto field_types = flat_vec(res_meta.selected_field_types);
// auto field_lengths = flat_vec(res_meta.selected_field_lengths);
auto bt = get_batch(res_meta,res);
vector<string> field_names = bt.field_names;
vector<int> field_types = bt.field_types;
vector<int> field_lengths = bt.field_lengths;
resraw.fieldNames = field_names;
for(unsigned int i=0;i<field_types.size();++i) {
for(unsigned int i=0;i<field_types.size();++i){
resraw.fieldTypes.push_back(static_cast<enum_field_types>(field_types[i]));
}
ResType rawtorestype = MygetResTypeFromLuaTable(false, &resraw);
......
debug/load.hh
View file @ 251a19a3
......@@ -131,9 +131,8 @@ static
ResType MygetResTypeFromLuaTable(bool isNULL,rawMySQLReturnValue *inRow = NULL,int in_last_insert_id = 0){
std::vector<std::string> names;
std::vector<enum_field_types> types;
std::vector<std::vector<Item *> > rows;
//return NULL restype
std::vector<std::vector<Item *>> rows;
//return NULL restype
if(isNULL){
return ResType(true,0,0,std::move(names),
std::move(types),std::move(rows));
......@@ -151,9 +150,7 @@ ResType MygetResTypeFromLuaTable(bool isNULL,rawMySQLReturnValue *inRow = NULL,i
}
rows.push_back(curTempRow);
}
return ResType(true, 0 ,
in_last_insert_id, std::move(names),
std::move(types), std::move(rows));
return ResType(true,0,in_last_insert_id,std::move(names),std::move(types),std::move(rows));
}
}
......@@ -323,17 +320,12 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
dec_names.push_back(rf.fieldCalled());
}
}
const unsigned int real_cols = dec_names.size();
std::vector<std::vector<Item *> > dec_rows(rows);
//real cols depends on plain text names.
for (unsigned int i = 0; i < rows; i++) {
dec_rows[i] = std::vector<Item *>(real_cols);
}
//
unsigned int col_index = 0;
for (unsigned int c = 0; c < cols; c++) {
......@@ -341,16 +333,13 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
if (rf.getIsSalt()) {
continue;
}
//the key is in fieldMeta
FieldMeta *const fm = rf.getOLK().key;
for (unsigned int r = 0; r < rows; r++) {
//
if (!fm || dbres.rows[r][c]->is_null()) {
dec_rows[r][col_index] = dbres.rows[r][c];
} else {
uint64_t salt = 0;
const int salt_pos = rf.getSaltPosition();
//read salt from remote datab for descrypting.
......@@ -360,7 +349,6 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
assert_s(!salt_item->null_value, "salt item is null");
salt = salt_item->value;
}
//specify fieldMeta, onion, and salt should be able to decrpyt
//peel onion
dec_rows[r][col_index] =
......@@ -369,13 +357,10 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
}
col_index++;
}
std::vector<enum_field_types> types;
for(auto item:dec_rows[0]){
types.push_back(item->field_type());
}
//resType is used befor and after descrypting.
return ResType(dbres.ok, dbres.affected_rows, dbres.insert_id,
std::move(dec_names),
......@@ -383,40 +368,6 @@ ResType decryptResults(const ResType &dbres, const ReturnMeta &rmeta) {
std::move(dec_rows));
}
struct meta_file{
string db,table;
int num_of_fields;
vector<string> field_types;
vector<int> field_lengths;
vector<string> field_names;
vector<int> choosen_onions;
void show(){
cout<<db<<endl;
cout<<table<<endl;
cout<<num_of_fields<<endl;
for(auto item:field_types){
cout<<item<<"\t";
}
cout<<endl;
for(auto item:field_lengths){
cout<<item<<"\t";
}
cout<<endl;
for(auto item:field_names){
cout<<item<<"\t";
}
cout<<endl;
for(auto item:choosen_onions){
cout<<item<<"\t";
}
cout<<endl;
}
};
static void init(){
std::string client="192.168.1.1:1234";
//one Wrapper per user.
......@@ -464,11 +415,8 @@ static void construct_insert(rawMySQLReturnValue & str,std::string table,std::ve
cur+="(";
for(unsigned int j=0u;j<str.rowValues[i].size();j++){
if(IS_NUM(str.fieldTypes[j])) {
// cout<<str.fieldTypes[j]<<endl;
cur+=str.rowValues[i][j]+=",";
// cout<<"isnum"<<endl;
}else{
//cur+=string("\"")+=str.rowValues[i][j]+="\",";
int len = str.rowValues[i][j].size();
mysql_real_escape_string(globalConn->get_conn(),globalEsp,
str.rowValues[i][j].c_str(),len);
......
debug/store.cc
View file @ 251a19a3
#include "debug/store.hh"
#include "debug/common.hh"
static void write_meta(rawMySQLReturnValue& resraw,std::vector<transField> &res,string db,string table){
metadata_files mf;
mf.set_db_table(db,table);
vector<vector<int>> selected_field_types;
vector<vector<int>> selected_field_lengths;
vector<vector<string>> selected_field_names;
vector<vector<int>> selected_onion_index;
vector<int> dec_onion_index;
vector<string> has_salt;
unsigned int type_index=0u,length_index=0u;
for(auto item:res){
vector<int> field_types;
vector<int> field_lengths;
vector<string> field_names=item.fields;
vector<string> field_names;
//only choosen fields
for(auto i:item.choosenOnions){
field_names.push_back(item.fields[i]);
}
if(item.hasSalt){
field_names.push_back(item.fields.back());
}
int onion_index = item.onionIndex;
for(auto tp:resraw.fieldTypes)
field_types.push_back(static_cast<int>(tp));
field_lengths = resraw.lengths;
for(unsigned int i=0u;i<field_names.size();i++){
field_types.push_back(static_cast<int>(resraw.fieldTypes[type_index]));
type_index++;
}
// field_lengths = resraw.lengths;
for(unsigned int i=0u;i<field_names.size();i++){
field_lengths.push_back(resraw.lengths[length_index]);
length_index++;
}
if(item.hasSalt){
has_salt.push_back("true");
}else has_salt.push_back("false");
......@@ -27,7 +43,7 @@ static void write_meta(rawMySQLReturnValue& resraw,std::vector<transField> &res,
selected_field_types.push_back(field_types);
selected_field_lengths.push_back(field_lengths);
selected_field_names.push_back(field_names);
dec_onion_index.push_back(onion_index);
dec_onion_index.push_back(onion_index);
}
mf.set_selected_field_types(selected_field_types);
mf.set_selected_field_lengths(selected_field_lengths);
......@@ -70,6 +86,7 @@ static void store(std::string db, std::string table){
std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
//transform the field so that selected onions can be used
std::vector<transField> res = getTransField(fms);
for(auto &item:res){
(void)item;
item.choosenOnions.push_back(0);
......@@ -80,7 +97,6 @@ static void store(std::string db, std::string table){
//write the tuples into files
write_raw_data_to_files(resraw,res,db,table);
}
int
main(int argc, char* argv[]){
init();
......
debug/store.hh
View file @ 251a19a3
......@@ -60,7 +60,6 @@ struct transField{
vector<int> choosenOnions;
//used to construct return meta
int onionIndex = 0;
int numOfOnions=0;
//onions
std::vector<std::string> fields;
std::vector<onion> onions;
......@@ -192,11 +191,6 @@ rawMySQLReturnValue executeAndGetResultRemote(Connect * curConn,std::string quer
return myRaw;
}
//first step of back
static
std::vector<FieldMeta *>
......@@ -217,9 +211,6 @@ getFieldMeta(SchemaInfo &schema,
}
}
static std::unique_ptr<SchemaInfo> myLoadSchemaInfo() {
std::unique_ptr<Connect> e_conn(Connect::getEmbedded(embeddedDir));
std::unique_ptr<SchemaInfo> schema(new SchemaInfo());
......@@ -283,7 +274,6 @@ std::string getTestQuery(SchemaInfo &schema, std::vector<transField> &tfds,
res += item.originalFm->getSaltName()+" , ";
}
}
res = res.substr(0,res.size()-2);
res = res + "FROM `"+db+std::string("`.`")+annotablename+"`";
return res;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment