Skip to content

P
Practical-Cryptdb
Commit e437ad25 authored by yiwenshao's avatar yiwenshao
Browse files
Options

remove old version of load and store

parent 8f810943
Show whitespace changes
Inline Side-by-side
Showing with 0 additions and 517 deletions
debug/load.cc deleted 100644 → 0
View file @ 8f810943
/*1. store data as column files, and restore data as plaintext insert query
* 2. plaintext insert query should be able to recover directly
* 3. should be able to used exsisting data to reduce the computation overhead(to be implemented)
*/
#include <stdlib.h>
#include <stdio.h>
#include <string>
#include <map>
#include <iostream>
#include <fstream>
#include <sstream>
#include <vector>
#include <algorithm>
#include "wrapper/reuse.hh"
#include "wrapper/common.hh"
using std::cout;
using std::cin;
using std::endl;
using std::vector;
using std::string;
using std::to_string;
static std::string embeddedDir="/t/cryt/shadow";
char * globalEsp=NULL;
int num_of_pipe = 4;
//global map, for each client, we have one WrapperState which contains ProxyState.
static std::map<std::string, WrapperState*> clients;
//This connection mimics the behaviour of MySQL-Proxy
Connect *globalConn;
/*for each field, convert the format to FieldMeta_Wrapper*/
static
void
construct_insert(rawMySQLReturnValue & str,std::string table,std::vector<std::string> &res){
std::string head = string("INSERT INTO `")+table+"` VALUES ";
int cnt = 0;
string cur=head;
for(unsigned int i=0u; i<str.rowValues.size();i++){
++cnt;
cur+="(";
for(unsigned int j=0u;j<str.rowValues[i].size();j++){
if(IS_NUM(str.fieldTypes[j])) {
cur+=str.rowValues[i][j]+=",";
}else{
int len = str.rowValues[i][j].size();
mysql_real_escape_string(globalConn->get_conn(),globalEsp,
str.rowValues[i][j].c_str(),len);
cur+=string("\"")+=string(globalEsp)+="\",";
}
}
cur.back()=')';
cur+=",";
if(cnt == num_of_pipe){
cnt = 0;
cur.back()=';';
res.push_back(cur);
cur=head;
}
}
if(cnt!=0){
cur.back()=';';
res.push_back(cur);
}
}
static void init(){
std::string client="192.168.1.1:1234";
//one Wrapper per user.
clients[client] = new WrapperState();
//Connect phase
ConnectionInfo ci("localhost", "root", "letmein",3306);
const std::string master_key = "113341234";
char *buffer;
if((buffer = getcwd(NULL, 0)) == NULL){
perror("getcwd error");
}
embeddedDir = std::string(buffer)+"/shadow";
SharedProxyState *shared_ps =
new SharedProxyState(ci, embeddedDir , master_key,
determineSecurityRating());
assert(0 == mysql_thread_init());
//we init embedded database here.
clients[client]->ps = std::unique_ptr<ProxyState>(new ProxyState(*shared_ps));
clients[client]->ps->safeCreateEmbeddedTHD();
//Connect end!!
globalConn = new Connect(ci.server, ci.user, ci.passwd, ci.port);
}
struct batch{
vector<string> field_names;
vector<int> field_types;
vector<int> field_lengths;
};
static
batch get_batch(metadata_files & input,std::vector<FieldMeta_Wrapper> &tfms){
vector<vector<int>> selected_field_types = input.get_selected_field_types();
vector<vector<int>> selected_field_lengths = input.get_selected_field_lengths();
vector<vector<string>> selected_field_names = input.get_selected_field_names();
vector<int> dec_onion_index = input.get_dec_onion_index();
vector<string> has_salt = input.get_has_salt();
vector<string> field_names;
vector<int> field_types;
vector<int> field_lengths;
for(auto i=0u;i<tfms.size();i++){
int index = dec_onion_index[i];
string dec_field_name = tfms[i].fields[index];
auto f = find(selected_field_names[i].begin(),selected_field_names[i].end(),dec_field_name);
assert(f!=selected_field_names[i].end());
int j = f - selected_field_names[i].begin();
if(has_salt[i]==string("true")){
field_names.push_back(selected_field_names[i][j]);
field_types.push_back(selected_field_types[i][j]);
field_lengths.push_back(selected_field_lengths[i][j]);
field_names.push_back(selected_field_names[i].back());
field_types.push_back(selected_field_types[i].back());
field_lengths.push_back(selected_field_lengths[i].back());
}else{
assert(1==2);
}
}
batch bt;
bt.field_names = field_names;
bt.field_types = field_types;
bt.field_lengths = field_lengths;
return bt;
}
static
std::shared_ptr<ReturnMeta> getReturnMeta(std::vector<FieldMeta*> fms,
std::vector<FieldMeta_Wrapper> &tfds){
assert(fms.size()==tfds.size());
std::shared_ptr<ReturnMeta> myReturnMeta = std::make_shared<ReturnMeta>();
int pos=0;
//construct OLK
for(auto i=0u;i<tfds.size();i++){
OLK curOLK(tfds[i].onions[tfds[i].onionIndex],
tfds[i].originalOm[tfds[i].onionIndex]->getSecLevel(),tfds[i].originalFm);
addToReturn(myReturnMeta.get(),pos++,curOLK,true,tfds[i].originalFm->getFieldName());
addSaltToReturn(myReturnMeta.get(),pos++);
}
return myReturnMeta;
}
static metadata_files load_meta(string db="tdb", string table="student", string filename="metadata.data"){
metadata_files mf;
mf.set_db(db);
mf.set_table(table);
mf.deserialize(filename);
return mf;
}
static void load_num(string filename,vector<string> &res){
std::ifstream infile(filename);
string line;
while(std::getline(infile,line)){
res.push_back(line);
}
infile.close();
}
static void load_string(string filename, vector<string> &res,unsigned long length){
char *buf = new char[length];
int fd = open(filename.c_str(),O_RDONLY);
while(read(fd,buf,length)!=0){
res.push_back(string(buf,length));
}
close(fd);
}
template<class T>
vector<T> flat_vec(vector<vector<T>> &input){
vector<T> res;
for(auto item:input){
for(auto i:item){
res.push_back(i);
}
}
return res;
}
/*load fields in plain string*/
static vector<vector<string>> load_table_fields(metadata_files & input,
std::vector<FieldMeta_Wrapper> &tfms) {
string db = input.get_db();
string table = input.get_table();
vector<vector<string>> res;
string prefix = string("data/")+db+"/"+table+"/";
auto bt = get_batch(input,tfms);
vector<string> field_names = bt.field_names;
vector<int> field_types = bt.field_types;
vector<int> field_lengths = bt.field_lengths;
vector<string> datafiles;
for(auto item:field_names){
datafiles.push_back(prefix+item);
}
for(unsigned int i=0u;i<field_names.size();i++){
vector<string> column;
if(IS_NUM(field_types[i])){
load_num(datafiles[i],column);
}else{
load_string(datafiles[i],column,field_lengths[i]);
}
for(unsigned int j=0u; j<column.size(); j++){
if(j>=res.size()){
res.push_back(vector<string>());
}
res[j].push_back(column[j]);
}
}
return res;
}
static ResType load_files(std::string db="tdb", std::string table="student"){
std::unique_ptr<SchemaInfo> schema = myLoadSchemaInfo(embeddedDir);
//get all the fields in the tables.
std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
auto res = FieldMeta_to_Wrapper(fms);
std::vector<FieldMetaTrans> res2;
for(auto fm:fms){
FieldMetaTrans tf;
res2.push_back(tf);
res2.back().trans(fm);
}
metadata_files res_meta = load_meta(db,table);
std::shared_ptr<ReturnMeta> rm = getReturnMeta(fms,res);
//why do we need this??
create_embedded_thd(0);
rawMySQLReturnValue resraw;
//load fields in the stored file
vector<vector<string>> res_field = load_table_fields(res_meta,res);
resraw.rowValues = res_field;
auto bt = get_batch(res_meta,res);
vector<string> field_names = bt.field_names;
vector<int> field_types = bt.field_types;
vector<int> field_lengths = bt.field_lengths;
resraw.fieldNames = field_names;
for(unsigned int i=0;i<field_types.size();++i){
resraw.fieldTypes.push_back(static_cast<enum_field_types>(field_types[i]));
}
ResType rawtorestype = rawMySQLReturnValue_to_ResType(false, &resraw);
auto finalresults = decryptResults(rawtorestype,*rm);
return finalresults;
}
int
main(int argc, char* argv[]){
init();
create_embedded_thd(0);
std::string db="tdb",table="student";
globalEsp = (char*)malloc(sizeof(char)*5000);
if(globalEsp==NULL){
printf("unable to allocate esp\n");
return 0;
}
/*load and decrypt*/
ResType res = load_files(db,table);
/*transform*/
rawMySQLReturnValue str;
transform_to_rawMySQLReturnValue(str,res);
std::vector<string> res_query;
/*get piped insert*/
construct_insert(str,table,res_query);
for(auto item:res_query){
cout<<item<<endl;
}
free(globalEsp);
/*the next step is to construct encrypted insert query*/
return 0;
}
debug/store.cc deleted 100644 → 0
View file @ 8f810943
#include "wrapper/common.hh"
#include "wrapper/reuse.hh"
static std::string embeddedDir="/t/cryt/shadow";
//global map, for each client, we have one WrapperState which contains ProxyState.
static std::map<std::string, WrapperState*> clients;
//This connection mimics the behaviour of MySQL-Proxy
Connect *globalConn;
//must be static, or we get "no previous declaration"
//execute the query and get the rawReturnVale, this struct can be copied.
static void init(){
std::string client="192.168.1.1:1234";
//one Wrapper per user.
clients[client] = new WrapperState();
//Connect phase
ConnectionInfo ci("localhost", "root", "letmein",3306);
const std::string master_key = "113341234";
char *buffer;
if((buffer = getcwd(NULL, 0)) == NULL){
perror("getcwd error");
}
embeddedDir = std::string(buffer)+"/shadow";
SharedProxyState *shared_ps =
new SharedProxyState(ci, embeddedDir , master_key,
determineSecurityRating());
assert(0 == mysql_thread_init());
//we init embedded database here.
clients[client]->ps = std::unique_ptr<ProxyState>(new ProxyState(*shared_ps));
clients[client]->ps->safeCreateEmbeddedTHD();
//Connect end!!
globalConn = new Connect(ci.server, ci.user, ci.passwd, ci.port);
}
//query for testing purposes
static
std::string getTestQuery(SchemaInfo &schema, std::vector<FieldMetaTrans> &tfds,
std::string db="tdb",std::string table="student1"){
std::string res = "SELECT ";
const std::unique_ptr<IdentityMetaKey> dbmeta_key(new IdentityMetaKey(db));
//get databaseMeta, search in the map
DatabaseMeta * dbm = schema.getChild(*dbmeta_key);
const TableMeta & tbm = *((*dbm).getChild(IdentityMetaKey(table)));
std::string annotablename = tbm.getAnonTableName();
//then a list of onion names
for(auto tf:tfds){
for(auto item : tf.getChoosenOnionName()){
res += item;
res += " , ";
}
if(tf.getHasSalt()){
res += tf.getSaltName() + " , ";
}
}
res = res.substr(0,res.size()-2);
res = res + "FROM `"+db+std::string("`.`")+annotablename+"`";
return res;
}
static void write_meta(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> &res,string db,string table){
metadata_files mf;
mf.set_db_table(db,table);
vector<vector<int>> selected_field_types;
vector<vector<int>> selected_field_lengths;
vector<vector<string>> selected_field_names;
vector<vector<int>> selected_onion_index;
vector<int> dec_onion_index;
vector<string> has_salt;
unsigned int type_index=0u,length_index=0u;
for(auto ft:res){
vector<int> field_types;
vector<int> field_lengths;
vector<string> field_names;
//only choosen fields
for(auto item:ft.getChoosenOnionName()){
field_names.push_back(item);
}
if(ft.getHasSalt()){
field_names.push_back(ft.getSaltName());
}
int onion_index = 0;
for(unsigned int i=0u;i<field_names.size();i++){
field_types.push_back(static_cast<int>(resraw.fieldTypes[type_index]));
type_index++;
}
for(unsigned int i=0u;i<field_names.size();i++){
field_lengths.push_back(resraw.lengths[length_index]);
length_index++;
}
if(ft.getHasSalt()){
has_salt.push_back("true");
}else has_salt.push_back("false");
selected_field_types.push_back(field_types);
selected_field_lengths.push_back(field_lengths);
selected_field_names.push_back(field_names);
dec_onion_index.push_back(onion_index);
}
mf.set_selected_field_types(selected_field_types);
mf.set_selected_field_lengths(selected_field_lengths);
mf.set_selected_field_names(selected_field_names);
mf.set_dec_onion_index(dec_onion_index);
mf.set_has_salt(has_salt);
mf.serialize();
}
static
void write_raw_data_to_files(rawMySQLReturnValue& resraw,std::vector<FieldMetaTrans> &res ,string db,string table){
//write metafiles
write_meta(resraw,res,db,table);
//write datafiles
write_row_data(resraw,db,table);
}
static void store(std::string db, std::string table){
std::unique_ptr<SchemaInfo> schema = myLoadSchemaInfo(embeddedDir);
//get all the fields in the tables
std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
//transform the field so that selected onions can be used
std::vector<FieldMetaTrans> res;
for(auto i=0u;i<fms.size();i++){
FieldMetaTrans ft;
res.push_back(ft);
res.back().trans(fms[i]);
std::vector<int> in{0};
//this is our strategy !!!!!
res.back().choose(in);
}
//generate the backup query and then fetch the tuples
std::string backup_query = getTestQuery(*schema,res,db,table);
rawMySQLReturnValue resraw = executeAndGetResultRemote(globalConn,backup_query);
//write the tuples into files
write_raw_data_to_files(resraw,res,db,table);
}
int
main(int argc, char* argv[]){
init();
std::string db="tdb",table="student";
store(db,table);
return 0;
}
debug/store_new.cc deleted 100644 → 0
View file @ 8f810943
#include "wrapper/common.hh"
#include "wrapper/reuse.hh"
static std::string embeddedDir="/t/cryt/shadow";
//global map, for each client, we have one WrapperState which contains ProxyState.
static std::map<std::string, WrapperState*> clients;
//This connection mimics the behaviour of MySQL-Proxy
Connect *globalConn;
//must be static, or we get "no previous declaration"
//execute the query and get the rawReturnVale, this struct can be copied.
static void init(){
std::string client="192.168.1.1:1234";
//one Wrapper per user.
clients[client] = new WrapperState();
//Connect phase
ConnectionInfo ci("localhost", "root", "letmein",3306);
const std::string master_key = "113341234";
char *buffer;
if((buffer = getcwd(NULL, 0)) == NULL){
perror("getcwd error");
}
embeddedDir = std::string(buffer)+"/shadow";
SharedProxyState *shared_ps =
new SharedProxyState(ci, embeddedDir , master_key,
determineSecurityRating());
assert(0 == mysql_thread_init());
//we init embedded database here.
clients[client]->ps = std::unique_ptr<ProxyState>(new ProxyState(*shared_ps));
clients[client]->ps->safeCreateEmbeddedTHD();
//Connect end!!
globalConn = new Connect(ci.server, ci.user, ci.passwd, ci.port);
}
static void write_meta(std::vector<FieldMetaTrans> &res,string db,string table){
vector<int> in{1,2,3,4};
for(auto &item:res){
item.setChoosenFieldLengths(in);
item.setChoosenFieldTypes(in);
}
TableMetaTrans mf(db,table,res);
mf.set_db_table(db,table);
mf.serialize();
TableMetaTrans mf2;
mf2.set_db_table(db,table);
mf2.deserialize();
return;
}
static
void write_raw_data_to_files(std::vector<FieldMetaTrans> &res ,string db,string table){
//write metafiles
write_meta(res,db,table);
}
static void store(std::string db, std::string table){
std::unique_ptr<SchemaInfo> schema = myLoadSchemaInfo(embeddedDir);
//get all the fields in the tables
std::vector<FieldMeta*> fms = getFieldMeta(*schema,db,table);
//transform the field so that selected onions can be used
std::vector<FieldMetaTrans> res;
for(auto i=0u;i<fms.size();i++){
FieldMetaTrans ft;
res.push_back(ft);
res.back().trans(fms[i]);
std::vector<int> in{0};
res.back().choose(in);
}
write_raw_data_to_files(res,db,table);
}
int
main(int argc, char* argv[]){
init();
std::string db="tdb",table="student";
store(db,table);
return 0;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment