Merge pull request #88 from basho/mv-sst-fadvise

Mv sst fadvise

db/db_impl.cc

@@ -1673,7 +1673,7 @@ Status DBImpl::MakeRoomForWrite(bool force) {
       uint64_t new_log_number = versions_->NewFileNumber();
       WritableFile* lfile = NULL;
       gPerfCounters->Inc(ePerfWriteNewMem);
-      s = env_->NewWritableFile(LogFileName(dbname_, new_log_number), &lfile);
+      s = env_->NewWriteOnlyFile(LogFileName(dbname_, new_log_number), &lfile);
       if (!s.ok()) {
         // Avoid chewing through file number space in a tight loop.
         versions_->ReuseFileNumber(new_log_number);
@@ -1820,7 +1820,7 @@ Status DB::Open(const Options& options, const std::string& dbname,
   if (s.ok()) {
     uint64_t new_log_number = impl->versions_->NewFileNumber();
     WritableFile* lfile;
-    s = options.env->NewWritableFile(LogFileName(dbname, new_log_number),
+    s = options.env->NewWriteOnlyFile(LogFileName(dbname, new_log_number),
                                      &lfile);
     if (s.ok()) {
       edit.SetLogNumber(new_log_number);

db/table_cache.cc

@@ -88,6 +88,7 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size, int level
   }
   else
   {
+      // (later, call SetForCompaction here too for files already in cache)
       gPerfCounters->Inc(ePerfTableCached);
   }   // else
   return s;
@@ -103,7 +104,7 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
   }
 
   Cache::Handle* handle = NULL;
-  Status s = FindTable(file_number, file_size, level, &handle);
+  Status s = FindTable(file_number, file_size, level, &handle, options.IsCompaction());
   if (!s.ok()) {
     return NewErrorIterator(s);
   }

include/leveldb/env.h

@@ -71,6 +71,7 @@ class Env {
   virtual Status NewWritableFile(const std::string& fname,
                                  WritableFile** result) = 0;
 
+  // Riak specific:
   // Derived from NewWritableFile.  One change: if the file exists,
   // move to the end of the file and continue writing.
   // new file.  On success, stores a pointer to the open file in
@@ -81,6 +82,16 @@ class Env {
   virtual Status NewAppendableFile(const std::string& fname,
                                    WritableFile** result) = 0;
 
+  // Riak specific:
+  // Allows for virtualized version of NewWritableFile that enables write 
+  // and close operations to execute on background threads 
+  //  (where platform supported).
+  //
+  // The returned file will only be accessed by one thread at a time.
+  virtual Status NewWriteOnlyFile(const std::string& fname,
+                                   WritableFile** result)
+  {return(NewWritableFile(fname, result));};
+
   // Returns true iff the named file exists.
   virtual bool FileExists(const std::string& fname) = 0;
 
@@ -233,6 +244,11 @@ class WritableFile {
   virtual Status Flush() = 0;
   virtual Status Sync() = 0;
 
+  // Riak specific:
+  // Provide hint where key/value data ends and metadata starts
+  //  in an .sst table file.
+  virtual void SetMetadataOffset(uint64_t) {};
+
  private:
   // No copying allowed
   WritableFile(const WritableFile&);
@@ -318,6 +334,9 @@ class EnvWrapper : public Env {
   Status NewAppendableFile(const std::string& f, WritableFile** r) {
     return target_->NewAppendableFile(f, r);
   }
+  Status NewWriteOnlyFile(const std::string& f, WritableFile** r) {
+    return target_->NewWriteOnlyFile(f, r);
+  }
   bool FileExists(const std::string& f) { return target_->FileExists(f); }
   Status GetChildren(const std::string& dir, std::vector<std::string>* r) {
     return target_->GetChildren(dir, r);

include/leveldb/perf_count.h

@@ -189,6 +189,7 @@ enum PerformanceCountersEnum
     ePerfThrottleBacklog1=64,//!< backlog at time of posting (level1+)
     ePerfThrottleCompacts1=65,//!< number of level 1+ compactions
 
+    ePerfBGWriteError=66,   //!< error in write/close, see syslog
 
     // must follow last index name to represent size of array
     //  (ASSUMES previous enum is highest value)

table/table_builder.cc

@@ -228,6 +228,9 @@ Status TableBuilder::Finish() {
   BlockHandle filter_block_handle, metaindex_block_handle, index_block_handle,
       sst_stats_block_handle;
 
+  // pass hint to Linux fadvise management
+  r->file->SetMetadataOffset(r->offset);
+
   // Write filter block
   if (ok() && r->filter_block != NULL) {
     WriteRawBlock(r->filter_block->Finish(), kNoCompression,

util/cache.cc

@@ -116,7 +116,7 @@ class HandleTable {
       LRUHandle* h = list_[i];
       while (h != NULL) {
         LRUHandle* next = h->next_hash;
-        Slice key = h->key();
+        /*Slice key =*/ h->key();  // eliminate unused var warning, but allow for side-effects
         uint32_t hash = h->hash;
         LRUHandle** ptr = &new_list[hash & (new_length - 1)];
         h->next_hash = *ptr;