Skip to content

Commit

Permalink
email && test
Browse files Browse the repository at this point in the history
  • Loading branch information
@MaTianmao
MaTianmao committed Dec 5, 2019
1 parent 1e1f9e7 commit ace3ce3
Show file tree
Hide file tree
Showing 11 changed files with 325 additions and 454 deletions.
7 changes: 4 additions & 3 deletions ART/N.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,8 @@ void N::helpFlush(std::atomic<N *> *n) {
N *now_node = n->load();
// printf("help\n");
if (N::isDirty(now_node)) {
printf("help, point to type is %d\n", ((BaseNode *)N::clearDirty(now_node))->type);
printf("help, point to type is %d\n",
((BaseNode *)N::clearDirty(now_node))->type);
flush_data((void *)n, sizeof(N *));
// clflush((char *)n, sizeof(N *), true, true);
n->compare_exchange_strong(now_node, N::clearDirty(now_node));
Expand Down Expand Up @@ -124,7 +125,7 @@ void N::writeLockOrRestart(bool &needRestart) {
return;
}
} while (!typeVersionLockObsolete->compare_exchange_weak(version,
version + 0b10));
version + 0b10));
}

void N::lockVersionOrRestart(uint64_t &version, bool &needRestart) {
Expand All @@ -133,7 +134,7 @@ void N::lockVersionOrRestart(uint64_t &version, bool &needRestart) {
return;
}
if (typeVersionLockObsolete->compare_exchange_strong(version,
version + 0b10)) {
version + 0b10)) {
version = version + 0b10;
} else {
needRestart = true;
Expand Down
2 changes: 1 addition & 1 deletion ART/N.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ class N : public BaseNode {
virtual ~N() {}

// 3b type 60b version 1b lock 1b obsolete
// std::atomic<uint64_t> typeVersionLockObsolete{0b100};
// std::atomic<uint64_t> typeVersionLockObsolete{0b100};
std::atomic<uint64_t> *typeVersionLockObsolete;
// version 1, unlocked, not obsolete
alignas(64) std::atomic<Prefix> prefix;
Expand Down
91 changes: 46 additions & 45 deletions ART/Tree.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ void *Tree::lookup(const Key *k) const {

typename Tree::OperationResults Tree::update(const Key *k) const {
EpochGuard NewEpoch;
restart:
restart:
bool needRestart = false;

N *node = nullptr;
Expand All @@ -131,59 +131,60 @@ typename Tree::OperationResults Tree::update(const Key *k) const {
auto v = node->getVersion(); // check version

switch (checkPrefix(node, k, level)) { // increases level
case CheckPrefixResult::NoMatch:
case CheckPrefixResult::NoMatch:
if (N::isObsolete(v) || !node->readUnlockOrRestart(v)) {
goto restart;
}
return OperationResults::NotFound;
case CheckPrefixResult::OptimisticMatch:
// fallthrough
case CheckPrefixResult::Match: {
// if (level >= k->getKeyLen()) {
// // key is too short
// // but it next fkey is 0
// return OperationResults::NotFound;
// }
nodeKey = k->fkey[level];

parentref = curref;
curref = N::getChild(nodeKey, node);
if (curref == nullptr)
nextNode = nullptr;
else
nextNode = N::clearDirty(curref->load());

if (nextNode == nullptr) {
if (N::isObsolete(v) || !node->readUnlockOrRestart(v)) {
// std::cout<<"retry\n";
goto restart;
}
return OperationResults::NotFound;
case CheckPrefixResult::OptimisticMatch:
// fallthrough
case CheckPrefixResult::Match: {
// if (level >= k->getKeyLen()) {
// // key is too short
// // but it next fkey is 0
// return OperationResults::NotFound;
// }
nodeKey = k->fkey[level];

parentref = curref;
curref = N::getChild(nodeKey, node);
if (curref == nullptr)
nextNode = nullptr;
else
nextNode = N::clearDirty(curref->load());

if (nextNode == nullptr) {
if (N::isObsolete(v) || !node->readUnlockOrRestart(v)) {
// std::cout<<"retry\n";
goto restart;
}
return OperationResults::NotFound;
}
if (N::isLeaf(nextNode)) {
node->lockVersionOrRestart(v, needRestart);
if (needRestart) {
// std::cout<<"retry\n";
goto restart;
}
if (N::isLeaf(nextNode)) {
node->lockVersionOrRestart(v, needRestart);
if (needRestart){
// std::cout<<"retry\n";
goto restart;
}

Leaf *leaf = N::getLeaf(nextNode);
if (!leaf->checkKey(k)) {
node->writeUnlock();
return OperationResults::NotFound;
}
// find key, update it
Leaf *newleaf = new (alloc_new_node_from_type(NTypes::Leaf))
Leaf(leaf->fkey, k->key_len, (char *)k->value, k->val_len);
// std::cout<<(int)(((BaseNode *)newleaf)->type)<<"\n";
flush_data((void *)newleaf, sizeof(Leaf));

N::change(node, nodeKey, N::setLeaf(newleaf));
Leaf *leaf = N::getLeaf(nextNode);
if (!leaf->checkKey(k)) {
node->writeUnlock();
return OperationResults::Success;
return OperationResults::NotFound;
}
level++;
// find key, update it
Leaf *newleaf = new (alloc_new_node_from_type(NTypes::Leaf))
Leaf(leaf->fkey, k->key_len, (char *)k->value, k->val_len);
// std::cout<<(int)(((BaseNode
// *)newleaf)->type)<<"\n";
flush_data((void *)newleaf, sizeof(Leaf));

N::change(node, nodeKey, N::setLeaf(newleaf));
node->writeUnlock();
return OperationResults::Success;
}
level++;
}
}
}
}
Expand Down
1 change: 1 addition & 0 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,4 @@ target_link_libraries(malloc_diff Indexes)
add_executable(unittest ${DIR_TEST_SRC})
target_link_libraries(unittest Indexes gtest)

add_executable(gen_mail script/gen_mail.cpp)
12 changes: 11 additions & 1 deletion benchmark/config.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ struct Config {
int num_threads;
unsigned long long init_keys;
int time;
int key_length;
bool share_memory;
float duration;

Expand Down Expand Up @@ -85,6 +86,7 @@ static void usage_exit(FILE *out) {
" -K --key_type : Key type : 0 (Integer) 1 (String) \n"
" -n --num_threads : Number of workers \n"
" -k --keys : Number of key-value pairs at begin\n"
" -L --key_length : Length of string key\n"
" -s --non_share_memory : Use different index instances among "
"different workers\n"
" -d --duration : Execution time\n"
Expand All @@ -104,6 +106,7 @@ static void parse_arguments(int argc, char *argv[], Config &state) {
state.key_type = Integer;
state.init_keys = 3000000;
state.time = 5;
state.key_length = 15;
state.share_memory = true;
state.duration = 1;
state.benchmark = READ_ONLY;
Expand All @@ -118,7 +121,7 @@ static void parse_arguments(int argc, char *argv[], Config &state) {
while (1) {
int idx = 0;
int c =
getopt_long(argc, argv, "f:t:K:n:k:sd:b:w:S:l:r:T:", opts, &idx);
getopt_long(argc, argv, "f:t:K:n:k:L:sd:b:w:S:l:r:T:", opts, &idx);

if (c == -1)
break;
Expand All @@ -142,6 +145,9 @@ static void parse_arguments(int argc, char *argv[], Config &state) {
case 'k':
state.init_keys = (1llu << atoi(optarg));
break;
case 'L':
state.key_length = atoi(optarg);
break;
case 's':
state.share_memory = false;
break;
Expand Down Expand Up @@ -172,7 +178,11 @@ static void parse_arguments(int argc, char *argv[], Config &state) {
usage_exit(stderr);
}
}
if(state.key_type == String){
std::cout<<"key length: "<<state.key_length<<"\n";
}
if (state.workload == ZIPFIAN)
std::cout << "zipfian skewness " << state.skewness << "\n";
std::cout<<"read ratio: "<<state.read_ratio<<"\n";
// state.report();
}
24 changes: 14 additions & 10 deletions benchmark/coordinator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ template <typename K, typename V, int size> class Coordinator {
res = art->insert(k);
break;
case REMOVE:
// art->insert(k);
// art->insert(k);
art->remove(k);
break;
case GET:
Expand Down Expand Up @@ -315,12 +315,14 @@ template <typename K, typename V, int size> class Coordinator {
break;
case REMOVE:
// first insert then remove
// if (conf.key_type == Integer) {
// // std::cout<<"insert key "<<d<<"\n";
// bt->btree_insert(d, value);
// } else if (conf.key_type == String) {
// bt->btree_insert((char *)s.c_str(), value);
// }
// if (conf.key_type == Integer) {
// // std::cout<<"insert
// key "<<d<<"\n"; bt->btree_insert(d,
// value);
// } else if (conf.key_type == String) {
// bt->btree_insert((char *)s.c_str(),
// value);
// }

if (conf.key_type == Integer) {
// std::cout<<"delete key "<<d<<"\n";
Expand Down Expand Up @@ -438,7 +440,8 @@ template <typename K, typename V, int size> class Coordinator {
"%s, benchmark %d, zipfian %.2lf\n",
(double)final_result.throughput / 1000000.0 / conf.duration,
conf.num_threads, (conf.type == PART) ? "ART" : "FF",
(conf.key_type == Integer) ? "Int" : "Str", conf.benchmark,(conf.workload == RANDOM) ? 0:conf.skewness);
(conf.key_type == Integer) ? "Int" : "Str", conf.benchmark,
(conf.workload == RANDOM) ? 0 : conf.skewness);

delete art;
delete[] pid;
Expand Down Expand Up @@ -473,7 +476,7 @@ template <typename K, typename V, int size> class Coordinator {
if (conf.key_type == Integer) {
long kk = benchmark->nextInitIntKey();
bt->btree_insert(kk, value);
std::cout << "insert key " << kk << "id: " << i << "\n";
std::cout << "insert key " << kk << "id: " << i << "\n";
} else if (conf.key_type == String) {
std::string s = benchmark->nextInitStrKey();
bt->btree_insert((char *)s.c_str(), value);
Expand Down Expand Up @@ -503,7 +506,8 @@ template <typename K, typename V, int size> class Coordinator {
"%s, benchmark %d, zipfian %.2lf\n",
(double)final_result.throughput / 1000000.0 / conf.duration,
conf.num_threads, (conf.type == PART) ? "ART" : "FF",
(conf.key_type == Integer) ? "Int" : "Str", conf.benchmark, (conf.workload == RANDOM) ? 0:conf.skewness);
(conf.key_type == Integer) ? "Int" : "Str", conf.benchmark,
(conf.workload == RANDOM) ? 0 : conf.skewness);

delete[] pid;
delete[] results;
Expand Down
55 changes: 21 additions & 34 deletions benchmark/generator.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,12 @@
#include "util.h"
#include "generator.h"
#include "util.h"

std::mutex ZipfWrapper::gen_mtx;
std::map<std::string, WorkloadFile*> ZipfWrapper::wf_map;
std::map<std::string, WorkloadFile *> ZipfWrapper::wf_map;

std::mutex dataset_mtx;

WorkloadFile::WorkloadFile(std::string filename)
{
WorkloadFile::WorkloadFile(std::string filename) {
std::ifstream fin;
fin.open(filename, std::ios::in | std::ios::binary);
fin.seekg(0, std::ios::end);
Expand Down Expand Up @@ -35,64 +34,52 @@ unsigned int hashfunc(uint32_t val) {
return hash;
}

int ZipfGenerator::randomInt()
{
int ZipfGenerator::randomInt() {
double d = rdm.randomDouble();

int low = 0, high = size;
while (low < high - 1)
{
while (low < high - 1) {
int mid = (low + high) / 2;
if (zipfs[mid] <= d && zipfs[mid + 1] > d)
{
low = mid; break;
}
else if (zipfs[mid] > d)
{
if (zipfs[mid] <= d && zipfs[mid + 1] > d) {
low = mid;
break;
} else if (zipfs[mid] > d) {
high = mid;
}
else
{
} else {
low = mid;
}
}
return hashfunc(low) % size;
}

void ZipfGenerator::init(double s, int inital)
{
void ZipfGenerator::init(double s, int inital) {
zipfs = new double[inital];
double sum = 0.0;
for (int i = 1; i < inital + 1; i++)
{
for (int i = 1; i < inital + 1; i++) {
zipfs[i - 1] = 1.0 / (float)pow((double)i, s);
sum += zipfs[i - 1];
}
zipfs[0] = 1.0 / sum;
for (int i = 1; i < inital; i++)
{
for (int i = 1; i < inital; i++) {
zipfs[i] = zipfs[i] / sum + zipfs[i - 1];
}
}

ZipfGenerator::ZipfGenerator(double s, int inital) : size(inital)
{
ZipfGenerator::ZipfGenerator(double s, int inital) : size(inital) {
init(s, inital);
}


ZipfWrapper::ZipfWrapper(double s, int inital){
ZipfWrapper::ZipfWrapper(double s, int inital) {
cursor = random();
std::string filename = get_file_name(s);
gen_mtx.lock();
if (wf_map.find(filename) == wf_map.end()){
if (access(filename.c_str(), 0)){
if (wf_map.find(filename) == wf_map.end()) {
if (access(filename.c_str(), 0)) {
std::cout << filename << " not exists, generate it now\n";
ZipfGenerator zipf(s, inital);
std::ofstream myfile;
myfile.open(filename, std::ios::out | std::ios::binary);
for (unsigned long long i = 0; i < inital * 16; i++)
{
for (unsigned long long i = 0; i < inital * 16; i++) {
int d = zipf.randomInt();
myfile.write((char *)&d, sizeof(int));
}
Expand All @@ -105,12 +92,12 @@ ZipfWrapper::ZipfWrapper(double s, int inital){
gen_mtx.unlock();
}

DataSet::DataSet(int size): data_size(size) {
std::string fn_str = get_file_name_str();
DataSet::DataSet(int size, int key_length) : data_size(size), key_len(key_length) {
std::string fn_str = get_file_name_str(key_len);
dataset_mtx.lock();
if (access(fn_str.c_str(), 0)) {
std::cout << fn_str << " not exist, generate it now\n";
RandomGenerator rdm;
RandomGenerator rdm(key_len);
std::ofstream myfile;
myfile.open(fn_str, std::ios::out);
for (unsigned long long i = 0; i < data_size; i++) {
Expand Down
Loading

0 comments on commit ace3ce3

Please sign in to comment.