Skip to content

Commit

Permalink
Cleanead up obsolete worker management code (XROOTD/SSI-based)
Browse files Browse the repository at this point in the history
  • Loading branch information
@iagaponenko
iagaponenko committed Nov 30, 2023
1 parent 1437b93 commit 55a1b2b
Show file tree
Hide file tree
Showing 48 changed files with 10 additions and 5,039 deletions.
20 changes: 0 additions & 20 deletions src/global/ResourceUnit.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,9 +83,6 @@ std::string ResourceUnit::path() const {
case UNKNOWN:
ss << _pathSep << "UNKNOWN_RESOURCE_UNIT";
break;
case WORKER:
ss << _workerId;
break;
default:
::abort();
break;
Expand All @@ -107,8 +104,6 @@ std::string ResourceUnit::prefix(UnitType const& r) {
return "chk";
case UNKNOWN:
return "UNKNOWN";
case WORKER:
return "worker";
case QUERY:
return "query";
case GARBAGE:
Expand All @@ -121,10 +116,6 @@ std::string ResourceUnit::makePath(int chunk, std::string const& db) {
return _pathSep + prefix(UnitType::DBCHUNK) + _pathSep + db + _pathSep + std::to_string(chunk);
}

std::string ResourceUnit::makeWorkerPath(std::string const& id) {
return _pathSep + prefix(UnitType::WORKER) + _pathSep + id;
}

void ResourceUnit::setAsDbChunk(std::string const& db, int chunk) {
_unitType = DBCHUNK;
_db = db;
Expand Down Expand Up @@ -173,17 +164,6 @@ void ResourceUnit::_setFromPath(std::string const& path) {
}
_chunk = t.tokenAsInt();
_ingestLeafAndKeys(t.token());
} else if (rTypeString == prefix(WORKER)) {
_unitType = WORKER;
if (_markGarbageIfDone(t)) {
return;
}
t.next();
_workerId = t.token();
if (_workerId.empty()) {
_unitType = GARBAGE;
return;
}
} else if (rTypeString == prefix(QUERY)) {
_unitType = QUERY;
if (!t.done()) {
Expand Down
7 changes: 1 addition & 6 deletions src/global/ResourceUnit.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ namespace lsst::qserv {
class ResourceUnit {
public:
class Checker;
enum UnitType { GARBAGE, DBCHUNK, UNKNOWN, WORKER, QUERY };
enum UnitType { GARBAGE, DBCHUNK, UNKNOWN, QUERY };

ResourceUnit() = default;
explicit ResourceUnit(std::string const& path);
Expand All @@ -61,7 +61,6 @@ class ResourceUnit {
UnitType unitType() const { return _unitType; }
std::string const& db() const { return _db; }
int chunk() const { return _chunk; }
std::string const& workerId() const { return _workerId; }

/// Lookup extended path variables (?k=val syntax)
std::string var(std::string const& key) const;
Expand All @@ -72,9 +71,6 @@ class ResourceUnit {
/// @return the path of the database/chunk resource
static std::string makePath(int chunk, std::string const& db);

/// @return the path of the worker-specific resource
static std::string makeWorkerPath(std::string const& id);

// Setup a path of a certain type.
void setAsDbChunk(std::string const& db, int chunk = DUMMY_CHUNK);

Expand All @@ -88,7 +84,6 @@ class ResourceUnit {
UnitType _unitType = UnitType::GARBAGE; //< Type of unit
std::string _db; //< for DBCHUNK type
int _chunk = -1; //< for DBCHUNK type
std::string _workerId; //< for WORKER type

typedef std::map<std::string, std::string> VarMap;
VarMap _vars; //< Key-value specifiers
Expand Down
7 changes: 0 additions & 7 deletions src/global/testResourceUnit.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,13 +81,6 @@ BOOST_AUTO_TEST_CASE(DbChunk) {
BOOST_CHECK_EQUAL(r[1].path(), "/chk/bar/968");
}

BOOST_AUTO_TEST_CASE(Worker) {
std::string const id = "worker-1";
ResourceUnit res("/worker/" + id);
BOOST_CHECK_EQUAL(res.unitType(), ResourceUnit::WORKER);
BOOST_CHECK_EQUAL(res.workerId(), id);
}

BOOST_AUTO_TEST_CASE(Query) {
ResourceUnit const res1("/query");
BOOST_CHECK_EQUAL(res1.unitType(), ResourceUnit::QUERY);
Expand Down
163 changes: 9 additions & 154 deletions src/proto/worker.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,171 +135,26 @@ message Result {
// Byte N+1, extent = ProtoHeader.size, Result msg
// (successive Result msgs indicated by size markers in previous Result msgs)


////////////////////////////////////////////////////////////////
// Protocol definition for the worker management commands
/////////////////////////////////////////////////////////////////
// Protocol definition for the query management requests. These
// requests do not require any response messages to be explicitly
// sent by workers.
//
// ATTENTION: each message sent or received must be preceeded by
// ATTENTION: each message sent to a worker must be preceeded by
// an int32 size (network-byte-ordered) word carrying a size
// of the message.
////////////////////////////////////////////////////////////////

// All requests sent to worker management resources should start with
// sending this message carrying an identifier of a command. Specific
// commands may require additional parameters which should be sent as
// separate messages (of the corresponding types).
message WorkerCommandH {
enum Command {
TEST_ECHO = 1; // Return back a value sent to the command processor.
ADD_CHUNK_GROUP = 2; // Add a group of collocated chunks.
REMOVE_CHUNK_GROUP = 3; // Remove a group of collocated chunks.
UPDATE_CHUNK_LIST = 4; // Update (rebuild and/or reload) the list of available chunks.
GET_CHUNK_LIST = 5; // Return a list of chunks known to a worker.
SET_CHUNK_LIST = 6; // Set a new list of chunks.
GET_STATUS = 7; // Return various status info on a worker.
GET_DATABASE_STATUS = 8; // Return various info on the worker database service.
GET_CONFIG = 9; // Return configuration parameters of the worker database service.
}
required Command command = 1;
}

// The completion status to be sent back with responses to the worker commands.
// The completion status to be sent back with responses to the query management requests.
message WorkerCommandStatus {
enum Code {
SUCCESS = 1; // The sccessful completion of a request.
INVALID = 2; // Invalid parameters of the request.
IN_USE = 3; // The request is rejected because one of the chunks is in use.
ERROR = 4; // An error occurred during command execution.
SUCCESS = 1; // The successful completion of a request.
ERROR = 2; // An error occurred during request execution.
}
optional Code code = 1 [default = SUCCESS];
optional Code code = 3 [default = SUCCESS];
optional string error = 2 [default = ""]; // Optional error message (depends on the code)
}

// This message must be sent after the command header to provide
// a service with a value to be echoed back in response to
// the 'TEST_ECHO' command.
message WorkerCommandTestEchoM {
required string value = 1; // The input string to be returned back by the service
}

// The message to be sent back in response to the 'TEST_ECHO' command
message WorkerCommandTestEchoR {
required WorkerCommandStatus status = 4; // Completion status of the operation
required string value = 3; // The original value returned by the operation
}

// The message type embedded into the relevant contexts below
message WorkerCommandChunk {
required string db = 1;
required uint32 chunk = 2;
optional uint32 use_count = 3 [default = 0]; // depends on a context
}

// This message must be sent after the command header for the 'ADD_CHUNK_GROUP'
// or 'REMOVE_CHUNK_GROUP' command to tell the service which chunks needs to be
// added or removed.
message WorkerCommandChunkGroupM {
required uint32 chunk = 1;
repeated string dbs = 2;
optional bool force = 3 [ default = false];
}

// The message to be sent back in response to the 'ADD_CHUNK_GROUP'
// or 'REMOVE_CHUNK_GROUP' commands.
message WorkerCommandChunkGroupR {
required WorkerCommandStatus status = 3; // Completion status of the operation
}

// This message must be sent after the command header for the 'UPDATE_CHUNK_LIST'
// command.
message WorkerCommandUpdateChunkListM {
required bool rebuild = 1; // Rebuild the list from actual tables existing in the database
required bool reload = 2; // Reload the new list into a worker
}

// The message to be sent back in response to the 'UPDATE_CHUNK_LIST'
// command.
message WorkerCommandUpdateChunkListR {
required WorkerCommandStatus status = 5; // Completion status of the operation
repeated WorkerCommandChunk added = 3; // Chunks that were added
repeated WorkerCommandChunk removed = 4; // Chunks that were removed
}

// The message to be sent back in response to the 'GET_CHUNK_LIST'
// command.
message WorkerCommandGetChunkListR {
required WorkerCommandStatus status = 4; // Completion status of the operation
repeated WorkerCommandChunk chunks = 3;
}

// This message must be sent after the command header for the 'SET_CHUNK_LIST'
// to tell the service which chunks needs to be set.
message WorkerCommandSetChunkListM {
repeated WorkerCommandChunk chunks = 1;
optional bool force = 2 [ default = false];
repeated string databases = 3; // The operation involves databases listed here
}

// The message to be sent back in response to the 'SET_CHUNK_LIST'
// command.
message WorkerCommandSetChunkListR {
required WorkerCommandStatus status = 4; // Completion status of the operation
repeated WorkerCommandChunk chunks = 3; // The previous list of chunks
}

// This message must be sent after the command header for the 'GET_STATUS'
// to customize a scope of the request.
message WorkerCommandGetStatusM {
// Include detailed info on the tasks
optional bool include_tasks = 1 [ default = false];

// Narrow a scope of the request that involves tasks (the above-define parameter
// 'include_tasks = true') to a specific collection of queries.
repeated uint64 query_ids = 2;

// An optional selector for the state(s) of tasks. If the collection
// is empty then the selector is disregarded. Values to be put into
// the collection are the integer representation of the following transient
// C++ enum type 'wbase::TaskState'. These values are expected to be
// validated and upconverted into the enum class by the worker.
repeated uint64 task_states = 3;

// An optional limit for the number of tasks to be reported.
// Note that no specific ordering for tasks is assumed when applying
// this limit.
optional uint32 max_tasks = 4 [ default = 0 ];
}

// The message to be sent back in response to the 'GET_STATUS' command
message WorkerCommandGetStatusR {
required WorkerCommandStatus status = 2; // Completion status of the operation
required string info = 1; // Status info serialized from a JSON object
}

// The message to be sent back in response to the 'GET_DATABASE_STATUS' command
//
message WorkerCommandGetDbStatusR {
required WorkerCommandStatus status = 1; // Completion status of the operation
required string info = 2; // Status info serialized from a JSON object
}

// The message to be sent back in response to the 'GET_CONFIG' command
//
message WorkerCommandGetConfigR {
required WorkerCommandStatus status = 1; // Completion status of the operation
required string info = 2; // Configuration info serialized from a JSON object
}

/////////////////////////////////////////////////////////////////
// Protocol definition for the query management requests. These
// requests do not require any response messages to be explicitly
// sent by workers.
//
// ATTENTION: each message sent to a worker must be preceeded by
// an int32 size (network-byte-ordered) word carrying a size
// of the message.
////////////////////////////////////////////////////////////////

message QueryManagement {
enum Operation {
CANCEL_AFTER_RESTART = 1; // Cancel older queries before the specified query (excluding that one).
Expand Down
111 changes: 0 additions & 111 deletions src/wpublish/AddChunkGroupCommand.cc
View file

This file was deleted.

Loading

0 comments on commit 55a1b2b

Please sign in to comment.