Skip to content

Commit

Permalink
feat(cpp): resolve issue #3 and #4, and support LoaderOptions: ignore…
Browse files Browse the repository at this point in the history
…_unknown_fields
  • Loading branch information
wenchy committed Oct 18, 2022
1 parent 3e21f21 commit 47bc231
Show file tree
Hide file tree
Showing 15 changed files with 606 additions and 130 deletions.
48 changes: 31 additions & 17 deletions _lab/cpp/src/demo/hub.pc.cc
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include "hub.pc.h"

#include <google/protobuf/stubs/logging.h>
#include <google/protobuf/stubs/status.h>
#include <google/protobuf/text_format.h>

#include <fstream>
Expand Down Expand Up @@ -36,8 +37,16 @@ bool Message2JSON(const google::protobuf::Message& message, std::string& json) {
return google::protobuf::util::MessageToJsonString(message, &json, options).ok();
}

bool JSON2Message(const std::string& json, google::protobuf::Message& message) {
auto status = google::protobuf::util::JsonStringToMessage(json, &message);
bool JSON2Message(const std::string& json, google::protobuf::Message& message,
const LoadOptions* options /* = nullptr */) {
google::protobuf::util::Status status;
if (options != nullptr) {
google::protobuf::util::JsonParseOptions parse_options;
parse_options.ignore_unknown_fields = options->ignore_unknown_fields;
status = google::protobuf::util::JsonStringToMessage(json, &message, parse_options);
} else {
status = google::protobuf::util::JsonStringToMessage(json, &message);
}
if (!status.ok()) {
g_err_msg = "failed to parse " + GetProtoName(message) + kJSONExt + ": " + status.ToString();
return false;
Expand All @@ -52,9 +61,9 @@ bool Text2Message(const std::string& text, google::protobuf::Message& message) {
}
return true;
}
bool Wire2Message(const std::string& wire, google::protobuf::Message& message) {
if (!message.ParseFromString(wire)) {
g_err_msg = "failed to parse " + GetProtoName(message) + kWireExt;
bool Bin2Message(const std::string& bin, google::protobuf::Message& message) {
if (!message.ParseFromString(bin)) {
g_err_msg = "failed to parse " + GetProtoName(message) + kBinExt;
return false;
}
return true;
Expand All @@ -77,18 +86,19 @@ bool ReadFile(const std::string& filename, std::string& content) {
return true;
}

bool LoadMessage(const std::string& dir, google::protobuf::Message& message, Format fmt) {
bool LoadMessage(const std::string& dir, google::protobuf::Message& message, Format fmt,
const LoadOptions* options /* = nullptr*/) {
message.Clear();
std::string basepath = dir + GetProtoName(message);
// TODO: support 3 formats: json, text, and wire.
// TODO: support 3 formats: json, text, and bin.
std::string content;
switch (fmt) {
case Format::kJSON: {
bool ok = ReadFile(basepath + kJSONExt, content);
if (!ok) {
return false;
}
return JSON2Message(content, message);
return JSON2Message(content, message, options);
}
case Format::kText: {
bool ok = ReadFile(basepath + kTextExt, content);
Expand All @@ -98,11 +108,11 @@ bool LoadMessage(const std::string& dir, google::protobuf::Message& message, For
return Text2Message(content, message);
}
case Format::kWire: {
bool ok = ReadFile(basepath + kWireExt, content);
bool ok = ReadFile(basepath + kBinExt, content);
if (!ok) {
return false;
}
return Wire2Message(content, message);
return Bin2Message(content, message);
}
default: {
g_err_msg = "unsupported format: %d" + static_cast<int>(fmt);
Expand All @@ -112,21 +122,23 @@ bool LoadMessage(const std::string& dir, google::protobuf::Message& message, For
}

bool StoreMessage(const std::string& dir, google::protobuf::Message& message, Format fmt) {
// TODO: write protobuf message to file, support 3 formats: json, text, and wire.
// TODO: write protobuf message to file, support 3 formats: json, text, and bin.
return false;
}

bool Hub::Load(const std::string& dir, Filter filter, Format fmt) {
auto msger_container = LoadNewMessagerContainer(dir, filter, fmt);
bool Hub::Load(const std::string& dir, Filter filter /* = nullptr */, Format fmt /* = Format::kJSON */,
const LoadOptions* options /* = nullptr */) {
auto msger_container = LoadNewMessagerContainer(dir, filter, fmt, options);
if (!msger_container) {
return false;
}
SetMessagerContainer(msger_container);
return true;
}

bool Hub::AsyncLoad(const std::string& dir, Filter filter, Format fmt) {
auto msger_container = LoadNewMessagerContainer(dir, filter, fmt);
bool Hub::AsyncLoad(const std::string& dir, Filter filter /* = nullptr */, Format fmt /* = Format::kJSON */,
const LoadOptions* options /* = nullptr */) {
auto msger_container = LoadNewMessagerContainer(dir, filter, fmt, options);
if (!msger_container) {
return false;
}
Expand All @@ -140,15 +152,17 @@ void Hub::InitScheduler() {
sched_->Current();
}

MessagerContainer Hub::LoadNewMessagerContainer(const std::string& dir, Filter filter, Format fmt) {
MessagerContainer Hub::LoadNewMessagerContainer(const std::string& dir, Filter filter /* = nullptr */,
Format fmt /* = Format::kJSON */,
const LoadOptions* options /* = nullptr */) {
// intercept protobuf error logs
auto old_handler = google::protobuf::SetLogHandler(ProtobufLogHandler);

auto msger_container = NewMessagerContainer(filter);
for (auto iter : *msger_container) {
auto&& name = iter.first;
ATOM_TRACE("loading %s", name.c_str());
bool ok = iter.second->Load(dir, fmt);
bool ok = iter.second->Load(dir, fmt, options);
if (!ok) {
ATOM_ERROR("load %s failed: %s", name.c_str(), GetErrMsg().c_str());
// restore to old protobuf log hanlder
Expand Down
27 changes: 18 additions & 9 deletions _lab/cpp/src/demo/hub.pc.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,26 @@ enum class Format {
};

constexpr const char* kJSONExt = ".json";
constexpr const char* kTextExt = ".text";
constexpr const char* kWireExt = ".wire";
constexpr const char* kTextExt = ".txt";
constexpr const char* kBinExt = ".bin";

static const std::string kEmpty = "";
const std::string& GetErrMsg();
struct LoadOptions {
// Whether to ignore unknown JSON fields during parsing.
// See
// https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.util.json_util#JsonParseOptions.
bool ignore_unknown_fields;
};

bool Message2JSON(const google::protobuf::Message& message, std::string& json);
bool JSON2Message(const std::string& json, google::protobuf::Message& message);
bool JSON2Message(const std::string& json, google::protobuf::Message& message, const LoadOptions* options = nullptr);
bool Text2Message(const std::string& text, google::protobuf::Message& message);
bool Wire2Message(const std::string& wire, google::protobuf::Message& message);
bool Bin2Message(const std::string& bin, google::protobuf::Message& message);
void ProtobufLogHandler(google::protobuf::LogLevel level, const char* filename, int line, const std::string& message);
const std::string& GetProtoName(const google::protobuf::Message& message);
bool LoadMessage(const std::string& dir, google::protobuf::Message& message, Format fmt = Format::kJSON);
bool LoadMessage(const std::string& dir, google::protobuf::Message& message, Format fmt = Format::kJSON,
const LoadOptions* options = nullptr);

namespace internal {
class Scheduler {
Expand Down Expand Up @@ -55,7 +62,7 @@ class Messager {
public:
virtual ~Messager() = default;
static const std::string& Name() { return kEmpty; };
virtual bool Load(const std::string& dir, Format fmt) = 0;
virtual bool Load(const std::string& dir, Format fmt, const LoadOptions* options = nullptr) = 0;

protected:
virtual bool ProcessAfterLoad() { return true; };
Expand All @@ -70,12 +77,14 @@ class Hub {
public:
/***** Synchronously Loading *****/
// Load messagers from dir using the specified format, and store them in MessagerContainer.
bool Load(const std::string& dir, Filter filter = nullptr, Format fmt = Format::kJSON);
bool Load(const std::string& dir, Filter filter = nullptr, Format fmt = Format::kJSON,
const LoadOptions* options = nullptr);

/***** Asynchronously Loading *****/
// Load configs into temp MessagerContainer, and you should call LoopOnce() in you app's main loop,
// in order to take the temp MessagerContainer into effect.
bool AsyncLoad(const std::string& dir, Filter filter, Format fmt = Format::kJSON);
bool AsyncLoad(const std::string& dir, Filter filter = nullptr, Format fmt = Format::kJSON,
const LoadOptions* options = nullptr);
int LoopOnce();
// You'd better initialize the scheduler in the main thread.
void InitScheduler();
Expand All @@ -96,7 +105,7 @@ class Hub {

private:
MessagerContainer LoadNewMessagerContainer(const std::string& dir, Filter filter = nullptr,
Format fmt = Format::kJSON);
Format fmt = Format::kJSON, const LoadOptions* options = nullptr);
MessagerContainer NewMessagerContainer(Filter filter = nullptr);
void SetMessagerContainer(MessagerContainer msger_container);
MessagerContainer GetMessagerContainerWithProvider() const;
Expand Down
107 changes: 104 additions & 3 deletions _lab/cpp/src/demo/item_conf.pc.cc
Original file line number Diff line number Diff line change
@@ -1,9 +1,110 @@
// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT.
// versions:
// - protoc-gen-cpp-tableau-loader v0.4.5
// - protoc v3.19.3
// source: item_conf.proto

#include "item_conf.pc.h"

namespace tableau {
const std::string ItemConf::kProtoName = "Item";
bool ItemConf::Load(const std::string& dir, Format fmt) {
bool ok = LoadMessage(dir, data_, fmt);
const std::string ItemConf::kProtoName = "ItemConf";

bool ItemConf::Load(const std::string& dir, Format fmt, const LoadOptions* options /* = nullptr */) {
bool ok = LoadMessage(dir, data_, fmt, options);
return ok ? ProcessAfterLoad() : false;
}

bool ItemConf::ProcessAfterLoad() {
// OrderedMap init.
for (auto&& item1 : data_.item_map()) {
ordered_map_[item1.first] = &item1.second;
}

// Index init.
for (auto&& item1 : data_.item_map()) {
index_item_map_[static_cast<int>(item1.second.type())].push_back(&item1.second);
}

for (auto&& item1 : data_.item_map()) {
for (auto&& item2 : item1.second.param_list()) {
index_item_info_map_[item2].push_back(&item1.second);
}
}

for (auto&& item1 : data_.item_map()) {
for (auto&& item2 : item1.second.ext_type_list()) {
index_item_ext_info_map_[static_cast<int>(item2)].push_back(&item1.second);
}
}

return true;
}

const protoconf::ItemConf::Item* ItemConf::Get(uint32_t key1) const {
auto iter = data_.item_map().find(key1);
if (iter == data_.item_map().end()) {
return nullptr;
}
return &iter->second;
}

const ItemConf::Item_OrderedMap* ItemConf::GetOrderedMap() const {
return &ordered_map_;
}

const ItemConf::Index_ItemInfoMap& ItemConf::FindItemInfo() const { return index_item_info_map_ ;}

const ItemConf::Index_ItemInfoVector* ItemConf::FindItemInfo(int32_t param_list) const {
auto iter = index_item_info_map_.find(param_list);
if (iter == index_item_info_map_.end()) {
return nullptr;
}
return &iter->second;
}

const protoconf::ItemConf::Item* ItemConf::FindFirstItemInfo(int32_t param_list) const {
auto conf = FindItemInfo(param_list);
if (conf == nullptr || conf->size() == 0) {
return nullptr;
}
return (*conf)[0];
}

const ItemConf::Index_ItemExtInfoMap& ItemConf::FindItemExtInfo() const { return index_item_ext_info_map_ ;}

const ItemConf::Index_ItemExtInfoVector* ItemConf::FindItemExtInfo(protoconf::FruitType ext_type_list) const {
auto iter = index_item_ext_info_map_.find(static_cast<int>(ext_type_list));
if (iter == index_item_ext_info_map_.end()) {
return nullptr;
}
return &iter->second;
}

const protoconf::ItemConf::Item* ItemConf::FindFirstItemExtInfo(protoconf::FruitType ext_type_list) const {
auto conf = FindItemExtInfo(ext_type_list);
if (conf == nullptr || conf->size() == 0) {
return nullptr;
}
return (*conf)[0];
}

const ItemConf::Index_ItemMap& ItemConf::FindItem() const { return index_item_map_ ;}

const ItemConf::Index_ItemVector* ItemConf::FindItem(protoconf::FruitType type) const {
auto iter = index_item_map_.find(static_cast<int>(type));
if (iter == index_item_map_.end()) {
return nullptr;
}
return &iter->second;
}

const protoconf::ItemConf::Item* ItemConf::FindFirstItem(protoconf::FruitType type) const {
auto conf = FindItem(type);
if (conf == nullptr || conf->size() == 0) {
return nullptr;
}
return (*conf)[0];
}


} // namespace tableau
57 changes: 55 additions & 2 deletions _lab/cpp/src/demo/item_conf.pc.h
Original file line number Diff line number Diff line change
@@ -1,23 +1,76 @@
// Code generated by protoc-gen-cpp-tableau-loader. DO NOT EDIT.
// versions:
// - protoc-gen-cpp-tableau-loader v0.4.5
// - protoc v3.19.3
// source: item_conf.proto

#pragma once
#include <string>

#include "hub.pc.h"
#include "protoconf/item_conf.pb.h"
#include "item_conf.pb.h"

namespace tableau {
class ItemConf : public Messager {
public:
static const std::string& Name() { return kProtoName; };
virtual bool Load(const std::string& dir, Format fmt) override;
virtual bool Load(const std::string& dir, Format fmt, const LoadOptions* options = nullptr) override;
const protoconf::ItemConf& Data() const { return data_; };

private:
virtual bool ProcessAfterLoad() override final;

public:
const protoconf::ItemConf::Item* Get(uint32_t key1) const;

private:
static const std::string kProtoName;
protoconf::ItemConf data_;

// OrderedMap accessers.
public:
using Item_OrderedMap = std::map<uint32_t, const protoconf::ItemConf::Item*>;
const Item_OrderedMap* GetOrderedMap() const;

private:
Item_OrderedMap ordered_map_;

// Index accessers.
public:
using Index_ItemVector = std::vector<const protoconf::ItemConf::Item*>;
using Index_ItemMap = std::unordered_map<int, Index_ItemVector>;
const Index_ItemMap& FindItem() const;
const Index_ItemVector* FindItem(protoconf::FruitType type) const;
const protoconf::ItemConf::Item* FindFirstItem(protoconf::FruitType type) const;

private:
Index_ItemMap index_item_map_;

public:
using Index_ItemInfoVector = std::vector<const protoconf::ItemConf::Item*>;
using Index_ItemInfoMap = std::unordered_map<int32_t, Index_ItemInfoVector>;
const Index_ItemInfoMap& FindItemInfo() const;
const Index_ItemInfoVector* FindItemInfo(int32_t param_list) const;
const protoconf::ItemConf::Item* FindFirstItemInfo(int32_t param_list) const;

private:
Index_ItemInfoMap index_item_info_map_;

public:
using Index_ItemExtInfoVector = std::vector<const protoconf::ItemConf::Item*>;
using Index_ItemExtInfoMap = std::unordered_map<int, Index_ItemExtInfoVector>;
const Index_ItemExtInfoMap& FindItemExtInfo() const;
const Index_ItemExtInfoVector* FindItemExtInfo(protoconf::FruitType ext_type_list) const;
const protoconf::ItemConf::Item* FindFirstItemExtInfo(protoconf::FruitType ext_type_list) const;

private:
Index_ItemExtInfoMap index_item_ext_info_map_;

};

} // namespace tableau

namespace protoconf {
// Here are some type aliases for easy use.
using ItemConfMgr = tableau::ItemConf;
} // namespace protoconf
Loading

0 comments on commit 47bc231

Please sign in to comment.