From 6cb71b237f0c8d76fa9f0f26393d1c4cb70fa571 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 17:47:17 +0000 Subject: [PATCH 01/18] add getfollowers rpc service --- services/follows/follows.proto | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/services/follows/follows.proto b/services/follows/follows.proto index 932fe953..2cf91041 100644 --- a/services/follows/follows.proto +++ b/services/follows/follows.proto @@ -45,7 +45,35 @@ message FollowResponse { string error = 2; } +message FollowUser { + string handle = 1; + string host = 2; + string display_name = 3; +} + +message GetFollowersRequest { + /* Eg. "admin", "weeura@samsoniuk.ua", "@jose" */ + string username; +} + +message GetFollowersResponse { + enum ResultType { + OK = 0; + ERROR = 1; + // More types can be added here. + } + + ResultType result_type = 1; + + /* Should only be set if result_type is not OK. */ + string error = 2; + + /* Should only be set if result_type is OK. */ + repeated FollowUser users = 3; +} + service Follows { rpc SendFollowRequest(LocalToAnyFollow) returns (FollowResponse); rpc ReceiveFollowRequest(ForeignToLocalFollow) returns (FollowResponse); + rpc GetFollowers(GetFollowersRequest) returns (GetFollowersResponse); } From feec12dcc4d25efd364b7a665b67075f9e634313 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 18:10:29 +0000 Subject: [PATCH 02/18] add follow FIND stuff to database proto --- services/database/database.proto | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/services/database/database.proto b/services/database/database.proto index 613f4d2a..13cc1771 100644 --- a/services/database/database.proto +++ b/services/database/database.proto @@ -90,10 +90,23 @@ message Follow { message DbFollowRequest { enum RequestType { INSERT = 0; + FIND = 1; } RequestType request_type = 1; + Follow entry = 2; + + /* If request_type is FIND: + * - If match.followed is set, then the service will return all followers + * for this user's ID. + * - If match.follower is set, the service will return all users this + * person follows. + * - If both match.followed and match.follower are set, the service will + * return an entry if this follow exists, and none otherwise. + * - If neither are set, all follows in the database will be returned. + */ + Follow match = 3; } message DbFollowResponse { @@ -104,6 +117,8 @@ message DbFollowResponse { ResultType result_type = 1; string error = 2; + + repeated Follow results = 3; } service Database { From 10debba27f14a61d7a9db62e2804ba978488488b Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 18:11:58 +0000 Subject: [PATCH 03/18] add find stuff to db follow_servicer --- services/database/follow_servicer.py | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/services/database/follow_servicer.py b/services/database/follow_servicer.py index 8ad47621..fbd37239 100644 --- a/services/database/follow_servicer.py +++ b/services/database/follow_servicer.py @@ -1,5 +1,7 @@ import sqlite3 +import util + import database_pb2 @@ -10,8 +12,26 @@ def __init__(self, db, logger): self._logger = logger self._follow_type_handlers = { database_pb2.DbFollowRequest.INSERT: self._follow_handle_insert, + database_pb2.DbFollowRequest.FIND: self._follow_handle_find, } + def _db_tuple_to_entry(self, tup, entry): + if len(tup) != 2: + self._logger.warning( + "Error converting tuple to Follow: " + + "Wrong number of elements " + str(tup)) + return False + try: + # You'd think there'd be a better way. + entry.follower = tup[0] + entry.followed = tup[1] + except Exception as e: + self._logger.warning( + "Error converting tuple to Follow: " + + str(e)) + return False + return True + def Follow(self, request, context): response = database_pb2.DbFollowResponse() self._follow_type_handlers[request.request_type](request, response) @@ -32,3 +52,21 @@ def _follow_handle_insert(self, req, resp): resp.error = str(e) return resp.result_type = database_pb2.DbFollowResponse.OK + + def _follow_handle_find(self, req, resp): + filter_clause, values = util.entry_to_filter(req.match) + try: + if not filter_clause: + res = self._db.execute('SELECT * FROM follows') + else: + res = self._db.execute( + 'SELECT * FROM posts WHERE ' + filter_clause, + *values) + except sqlite3.Error as e: + resp.result_type = database_pb2.DbFollowsResponse.ERROR + resp.error = str(e) + return + resp.result_type = database_pb2.DbFollowsResponse.OK + for tup in res: + if not self._db_tuple_to_entry(tup, resp.results.add()): + del resp.results[-1] From 33a49cf856ae489bc61b4c7ce69618569b7d376a Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 18:12:28 +0000 Subject: [PATCH 04/18] add stuff to query follows to follows service util --- services/follows/util.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/services/follows/util.py b/services/follows/util.py index 197c5ba3..9084a6e5 100644 --- a/services/follows/util.py +++ b/services/follows/util.py @@ -50,7 +50,8 @@ def get_user_from_db(self, handle, host, attempt_number=0): if len(find_resp.results) == 0: self._logger.warning('No user %s@%s found', handle, host) self._create_user_in_db(user_entry) - return self.get_user_from_db(handle, host, attempt_number=attempt_number+1) + return self.get_user_from_db(handle, host, + attempt_number=attempt_number+1) elif len(find_resp.results) == 1: self._logger.debug('Found user %s@%s from database', handle, host) return find_resp.results[0] @@ -72,6 +73,24 @@ def create_follow_in_db(self, follower_id, followed_id): ) follow_resp = self._db.Follow(follow_req) if follow_resp.result_type == database_pb2.DbFollowResponse.ERROR: - self._logger.error('Could not add follow to database: %s', follow_resp.error) + self._logger.error('Could not add follow to database: %s', + follow_resp.error) return follow_resp + def get_follows(self, follower_id=None, followed_id=None): + self._logger.debug('Finding follows ', + ('*' if follower_id is None else str(follower_id)) + ('*' if followed_id is None else str(followed_id)) + follow_entry = database_pb2.Follow( + follower=follower_id, + followed=followed_id + ) + follow_req = database_pb2.DbFollowRequest( + request_type=database_pb2.DbFollowRequest.FIND, + entry=follow_entry + ) + follow_resp = self._db.Follow(follow_req) + if follow_resp.result_type == database_pb2.DbFollowResponse.ERROR: + self._logger.error('Could not add follow to database: %s', + follow_resp.error) + return follow_Resp From 194484ee774e2b52bdc829338c7a6d77c9b8f1ca Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 18:14:02 +0000 Subject: [PATCH 05/18] Add get_followers follows servicer --- services/follows/get_followers.py | 46 +++++++++++++++++++++++++++++++ services/follows/servicer.py | 4 +++ 2 files changed, 50 insertions(+) create mode 100644 services/follows/get_followers.py diff --git a/services/follows/get_followers.py b/services/follows/get_followers.py new file mode 100644 index 00000000..331610a4 --- /dev/null +++ b/services/follows/get_followers.py @@ -0,0 +1,46 @@ +import database_pb2 +import follows_pb2 + + +class SendFollowServicer: + + def __init__(self, logger, util, database_stub): + self._logger = logger + self._util = util + self._database_stub = database_stub + + def GetFollowers(self, request, context): + self._logger.debug('List of followers of %s requested', request.username) + resp = follows_pb2.GetFollowersReponse() + + handle, host = self._util.parse_username( + request.username) + if handle is None and host is None: + resp.result_type = follows_pb2.GetFollowersResponse.ERROR + resp.error = 'Could not parse queried username' + return resp + + # Get user IDs for follow. + user_entry = self._util.get_user_from_db(handle, host) + if user_entry is None: + error = 'Could not find or create user {}@{}'.format(from_handle, + from_instance) + self._logger.error(error) + resp.result_type = follows_pb2.GetFollowersResponse.ERROR + resp.error = error + return resp + + user_id = user_entry.global_id + + follow_resp = self._util.create_follow_in_db(follower_entry.global_id, + followed_entry.global_id) + if follow_resp.result_type == database_pb2.DbFollowResponse.ERROR: + self._logger.error('Error creating follow: %s', follow_resp.error) + resp.result_type = follows_pb2.FollowResponse.ERROR + resp.error = 'Could not add requested follow to database' + return resp + + # TODO(#61): Send Follow activity if followed user is not local. + + resp.result_type = follows_pb2.FollowResponse.OK + return resp diff --git a/services/follows/servicer.py b/services/follows/servicer.py index 0f72d6a8..6468a3fc 100644 --- a/services/follows/servicer.py +++ b/services/follows/servicer.py @@ -1,3 +1,4 @@ +from get_followers import GetFollowersReceiver from receive_follow import ReceiveFollowServicer from send_follow import SendFollowServicer @@ -14,3 +15,6 @@ def __init__(self, logger, util, database_stub): self.SendFollowRequest = send_servicer.SendFollowRequest rec_servicer = ReceiveFollowServicer(logger, util, database_stub) self.ReceiveFollowRequest = rec_servicer.ReceiveFollowRequest + + get_followers_receiver = GetFollowersReceiver(logger, util, database_stub) + self.GetFollowers = get_followers_receiver.GetFollowers From b51e6498d5b3ccf249a867d81918c6a282dfecaf Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 18:39:43 +0000 Subject: [PATCH 06/18] allow getting user with an id instead of host and handle --- services/follows/util.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/services/follows/util.py b/services/follows/util.py index 9084a6e5..b9309683 100644 --- a/services/follows/util.py +++ b/services/follows/util.py @@ -33,14 +33,20 @@ def _create_user_in_db(self, entry): # TODO(iandioch): Respond to errors. return insert_resp - def get_user_from_db(self, handle, host, attempt_number=0): + def get_user_from_db(self, + handle=None, + host=None, + global_id=None, + attempt_number=0): if attempt_number > MAX_FIND_RETRIES: self._logger.error('Retried query too many times.') return None - self._logger.debug('User %s@%s requested from database', handle, host) + self._logger.debug('User %s@%s (id %s) requested from database', + handle, host, global_id) user_entry = database_pb2.UsersEntry( handle=handle, - host=host + host=host, + global_id=global_id ) find_req = database_pb2.UsersRequest( request_type=database_pb2.UsersRequest.FIND, @@ -48,16 +54,22 @@ def get_user_from_db(self, handle, host, attempt_number=0): ) find_resp = self._db.Users(find_req) if len(find_resp.results) == 0: - self._logger.warning('No user %s@%s found', handle, host) + self._logger.warning('No user %s@%s (id %s) found', + handle, host, global_id) + if global_id is not None: + # Should not try to create a user and hope it has this ID. + return None self._create_user_in_db(user_entry) return self.get_user_from_db(handle, host, attempt_number=attempt_number+1) elif len(find_resp.results) == 1: - self._logger.debug('Found user %s@%s from database', handle, host) + self._logger.debug('Found user %s@%s (id %s) from database', + handle, host, global_id) return find_resp.results[0] else: - self._logger.error('> 1 user found in database for %s@%s' + - ', returning first one.', handle, host) + self._logger.error('> 1 user found in database for %s@%s (id %s)' + + ', returning first one.', + handle, host, global_id) return find_resp.results[0] def create_follow_in_db(self, follower_id, followed_id): @@ -93,4 +105,4 @@ def get_follows(self, follower_id=None, followed_id=None): if follow_resp.result_type == database_pb2.DbFollowResponse.ERROR: self._logger.error('Could not add follow to database: %s', follow_resp.error) - return follow_Resp + return follow_resp From 6788a403058703ad8ececa534f0cb2854c900302 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 18:40:02 +0000 Subject: [PATCH 07/18] update send_follow to match new util func --- services/follows/send_follow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/follows/send_follow.py b/services/follows/send_follow.py index 76839bcb..6dc601ae 100644 --- a/services/follows/send_follow.py +++ b/services/follows/send_follow.py @@ -28,7 +28,7 @@ def SendFollowRequest(self, request, context): # Get user IDs for follow. follower_entry = self._util.get_user_from_db( - from_handle, from_instance) + handle=from_handle, host=from_instance) if follower_entry is None: error = 'Could not find or create user {}@{}'.format(from_handle, from_instance) @@ -36,7 +36,8 @@ def SendFollowRequest(self, request, context): resp.result_type = follows_pb2.FollowResponse.ERROR resp.error = error return resp - followed_entry = self._util.get_user_from_db(to_handle, to_instance) + followed_entry = self._util.get_user_from_db(handle=to_handle, + host=to_instance) if followed_entry is None: error = 'Could not find or create user {}@{}'.format(to_handle, to_instance) From 61adbfa0bfb862d455c61f130b4fb07285d2f759 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 18:45:48 +0000 Subject: [PATCH 08/18] add get_followers impl --- services/follows/get_followers.py | 36 ++++++++++++------------------- services/follows/util.py | 12 +++++++++++ 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/services/follows/get_followers.py b/services/follows/get_followers.py index 331610a4..c8cec87b 100644 --- a/services/follows/get_followers.py +++ b/services/follows/get_followers.py @@ -10,7 +10,8 @@ def __init__(self, logger, util, database_stub): self._database_stub = database_stub def GetFollowers(self, request, context): - self._logger.debug('List of followers of %s requested', request.username) + self._logger.debug('List of followers of %s requested', + request.username) resp = follows_pb2.GetFollowersReponse() handle, host = self._util.parse_username( @@ -20,27 +21,18 @@ def GetFollowers(self, request, context): resp.error = 'Could not parse queried username' return resp - # Get user IDs for follow. - user_entry = self._util.get_user_from_db(handle, host) - if user_entry is None: - error = 'Could not find or create user {}@{}'.format(from_handle, - from_instance) - self._logger.error(error) - resp.result_type = follows_pb2.GetFollowersResponse.ERROR - resp.error = error - return resp - - user_id = user_entry.global_id - - follow_resp = self._util.create_follow_in_db(follower_entry.global_id, - followed_entry.global_id) - if follow_resp.result_type == database_pb2.DbFollowResponse.ERROR: - self._logger.error('Error creating follow: %s', follow_resp.error) - resp.result_type = follows_pb2.FollowResponse.ERROR - resp.error = 'Could not add requested follow to database' - return resp - - # TODO(#61): Send Follow activity if followed user is not local. + following_ids = util.get_follows(followed_id) + + for _id in following_ids: + user = self._util.get_user_from_db(global_id = _id) + if user is None: + self._logger.warning('Could not find following user for id %d', + _id) + continue + if self._util.convert_db_user_to_follow_user(user, + resp.results.add()): + self._logger.warning('Could not conver user %s@%s to ' + + 'FollowUser', user.handle, user.host) resp.result_type = follows_pb2.FollowResponse.OK return resp diff --git a/services/follows/util.py b/services/follows/util.py index b9309683..d43d2fd5 100644 --- a/services/follows/util.py +++ b/services/follows/util.py @@ -1,3 +1,4 @@ +import follows_pb2 import database_pb2 MAX_FIND_RETRIES = 3 @@ -106,3 +107,14 @@ def get_follows(self, follower_id=None, followed_id=None): self._logger.error('Could not add follow to database: %s', follow_resp.error) return follow_resp + + def convert_db_user_to_follow_user(self, db_user, follow_user): + try: + follow_user.handle = db_user.handle + follow_user.host = db_user.host + follow_user.display_name = db_user.display_name + except Exception as e: + self._logger.warning('Error converting db user to follow user: ' + + str(e)) + return False + return True From 380adb8b8c5921c00c9c4abe17ed752663215ee6 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 19:00:35 +0000 Subject: [PATCH 09/18] modify proto rpc def. to allow for followers & following --- services/follows/follows.proto | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/follows/follows.proto b/services/follows/follows.proto index 2cf91041..2b3decc2 100644 --- a/services/follows/follows.proto +++ b/services/follows/follows.proto @@ -51,12 +51,12 @@ message FollowUser { string display_name = 3; } -message GetFollowersRequest { +message GetFollowsRequest { /* Eg. "admin", "weeura@samsoniuk.ua", "@jose" */ string username; } -message GetFollowersResponse { +message GetFollowsResponse { enum ResultType { OK = 0; ERROR = 1; @@ -75,5 +75,6 @@ message GetFollowersResponse { service Follows { rpc SendFollowRequest(LocalToAnyFollow) returns (FollowResponse); rpc ReceiveFollowRequest(ForeignToLocalFollow) returns (FollowResponse); - rpc GetFollowers(GetFollowersRequest) returns (GetFollowersResponse); + rpc GetFollowers(GetFollowsRequest) returns (GetFollowsResponse); + rpc GetFollowing(GetFollowsRequest) returns (GetFollowsResponse) } From 75f857537fb1057066c80a9b45eca2fd97cce1a2 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 19:00:52 +0000 Subject: [PATCH 10/18] add GetFollowing impl! --- services/follows/get_followers.py | 47 +++++++++++++++++++++++++------ services/follows/servicer.py | 5 ++-- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/services/follows/get_followers.py b/services/follows/get_followers.py index c8cec87b..db6c7df4 100644 --- a/services/follows/get_followers.py +++ b/services/follows/get_followers.py @@ -1,3 +1,5 @@ +from enum import Enum + import database_pb2 import follows_pb2 @@ -8,25 +10,48 @@ def __init__(self, logger, util, database_stub): self._logger = logger self._util = util self._database_stub = database_stub + self.RequestType = Enum('FOLLOWING', 'FOLLOWER') - def GetFollowers(self, request, context): - self._logger.debug('List of followers of %s requested', - request.username) - resp = follows_pb2.GetFollowersReponse() + def _get_follows(self, request, context, request_type): + if request_type == self.RequestType.FOLLOWERS: + self._logger.debug('List of followers of %s requested', + request.username) + else: + self._logger.debug('List of users %s is following requested', + request.username) + resp = follows_pb2.GetFollowsReponse() + # Parse input username handle, host = self._util.parse_username( request.username) if handle is None and host is None: - resp.result_type = follows_pb2.GetFollowersResponse.ERROR + resp.result_type = follows_pb2.GetFollowsResponse.ERROR resp.error = 'Could not parse queried username' return resp - following_ids = util.get_follows(followed_id) + # Get user obj associated with given user handle & host from database + user_entry = self._util.get_user_from_db(handle, host) + if user_entry is None: + error = 'Could not find or create user {}@{}'.format(from_handle, + from_instance) + self._logger.error(error) + resp.result_type = follows_pb2.GetFollowersResponse.ERROR + resp.error = error + return resp + user_id = user_entry.global_id + # Get followers/followings for this user. + following_ids = None + if request_type == self.RequestType.FOLLOWERS: + following_ids = util.get_follows(followed_id=user_id) + else: + following_ids = util.get_follows(follower_id=user_id) + + # Convert other following users and add to output proto. for _id in following_ids: user = self._util.get_user_from_db(global_id = _id) if user is None: - self._logger.warning('Could not find following user for id %d', + self._logger.warning('Could not find user for id %d', _id) continue if self._util.convert_db_user_to_follow_user(user, @@ -34,5 +59,11 @@ def GetFollowers(self, request, context): self._logger.warning('Could not conver user %s@%s to ' + 'FollowUser', user.handle, user.host) - resp.result_type = follows_pb2.FollowResponse.OK + resp.result_type = follows_pb2.GetFollowsResponse.OK return resp + + def GetFollowing(self, request, context): + return self._get_follows(request, context, self.RequestType.FOLLOWING) + + def GetFollowers(self, request, context): + return self._get_follows(request, context, self.RequestType.FOLLOWER) diff --git a/services/follows/servicer.py b/services/follows/servicer.py index 6468a3fc..ac64833c 100644 --- a/services/follows/servicer.py +++ b/services/follows/servicer.py @@ -16,5 +16,6 @@ def __init__(self, logger, util, database_stub): rec_servicer = ReceiveFollowServicer(logger, util, database_stub) self.ReceiveFollowRequest = rec_servicer.ReceiveFollowRequest - get_followers_receiver = GetFollowersReceiver(logger, util, database_stub) - self.GetFollowers = get_followers_receiver.GetFollowers + get_follows_receiver = GetFollowsReceiver(logger, util, database_stub) + self.GetFollowers = get_follows_receiver.GetFollowers + self.GetFollowing = get_follows_receiver.GetFollowing From 083c7c51a4cb710b01c8355fed8dd907bba66ad9 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 19:02:38 +0000 Subject: [PATCH 11/18] fmt --- services/follows/follows.proto | 2 +- services/follows/get_followers.py | 4 ++-- services/follows/util.py | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/services/follows/follows.proto b/services/follows/follows.proto index 2b3decc2..7e0c030d 100644 --- a/services/follows/follows.proto +++ b/services/follows/follows.proto @@ -76,5 +76,5 @@ service Follows { rpc SendFollowRequest(LocalToAnyFollow) returns (FollowResponse); rpc ReceiveFollowRequest(ForeignToLocalFollow) returns (FollowResponse); rpc GetFollowers(GetFollowsRequest) returns (GetFollowsResponse); - rpc GetFollowing(GetFollowsRequest) returns (GetFollowsResponse) + rpc GetFollowing(GetFollowsRequest) returns (GetFollowsResponse); } diff --git a/services/follows/get_followers.py b/services/follows/get_followers.py index db6c7df4..9f770464 100644 --- a/services/follows/get_followers.py +++ b/services/follows/get_followers.py @@ -29,7 +29,7 @@ def _get_follows(self, request, context, request_type): resp.error = 'Could not parse queried username' return resp - # Get user obj associated with given user handle & host from database + # Get user obj associated with given user handle & host from database user_entry = self._util.get_user_from_db(handle, host) if user_entry is None: error = 'Could not find or create user {}@{}'.format(from_handle, @@ -49,7 +49,7 @@ def _get_follows(self, request, context, request_type): # Convert other following users and add to output proto. for _id in following_ids: - user = self._util.get_user_from_db(global_id = _id) + user = self._util.get_user_from_db(global_id=_id) if user is None: self._logger.warning('Could not find user for id %d', _id) diff --git a/services/follows/util.py b/services/follows/util.py index d43d2fd5..8c58fa1e 100644 --- a/services/follows/util.py +++ b/services/follows/util.py @@ -62,7 +62,7 @@ def get_user_from_db(self, return None self._create_user_in_db(user_entry) return self.get_user_from_db(handle, host, - attempt_number=attempt_number+1) + attempt_number=attempt_number + 1) elif len(find_resp.results) == 1: self._logger.debug('Found user %s@%s (id %s) from database', handle, host, global_id) @@ -94,15 +94,15 @@ def get_follows(self, follower_id=None, followed_id=None): self._logger.debug('Finding follows ', ('*' if follower_id is None else str(follower_id)) ('*' if followed_id is None else str(followed_id)) - follow_entry = database_pb2.Follow( + follow_entry=database_pb2.Follow( follower=follower_id, followed=followed_id ) - follow_req = database_pb2.DbFollowRequest( + follow_req=database_pb2.DbFollowRequest( request_type=database_pb2.DbFollowRequest.FIND, entry=follow_entry ) - follow_resp = self._db.Follow(follow_req) + follow_resp=self._db.Follow(follow_req) if follow_resp.result_type == database_pb2.DbFollowResponse.ERROR: self._logger.error('Could not add follow to database: %s', follow_resp.error) @@ -110,9 +110,9 @@ def get_follows(self, follower_id=None, followed_id=None): def convert_db_user_to_follow_user(self, db_user, follow_user): try: - follow_user.handle = db_user.handle - follow_user.host = db_user.host - follow_user.display_name = db_user.display_name + follow_user.handle=db_user.handle + follow_user.host=db_user.host + follow_user.display_name=db_user.display_name except Exception as e: self._logger.warning('Error converting db user to follow user: ' + str(e)) From e67d053cc8646c4105bac2eb0fa25cb04a6e5e75 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 22:17:20 +0000 Subject: [PATCH 12/18] find using match instead of entry --- services/follows/util.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/services/follows/util.py b/services/follows/util.py index 8c58fa1e..d351c9fc 100644 --- a/services/follows/util.py +++ b/services/follows/util.py @@ -92,15 +92,15 @@ def create_follow_in_db(self, follower_id, followed_id): def get_follows(self, follower_id=None, followed_id=None): self._logger.debug('Finding follows ', - ('*' if follower_id is None else str(follower_id)) - ('*' if followed_id is None else str(followed_id)) + ('*' if (follower_id is None) else str(follower_id)), + ('*' if (followed_id is None) else str(followed_id))) follow_entry=database_pb2.Follow( follower=follower_id, followed=followed_id ) follow_req=database_pb2.DbFollowRequest( request_type=database_pb2.DbFollowRequest.FIND, - entry=follow_entry + match=follow_entry ) follow_resp=self._db.Follow(follow_req) if follow_resp.result_type == database_pb2.DbFollowResponse.ERROR: @@ -109,6 +109,7 @@ def get_follows(self, follower_id=None, followed_id=None): return follow_resp def convert_db_user_to_follow_user(self, db_user, follow_user): + self._logger.warning('Trying to convert %s %s', db_user.handle, db_user.host) try: follow_user.handle=db_user.handle follow_user.host=db_user.host From 21c661f49c32dec783e8b8bcda6480e93c63b14d Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 22:23:11 +0000 Subject: [PATCH 13/18] Fix enum in get_followers --- services/follows/get_followers.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/services/follows/get_followers.py b/services/follows/get_followers.py index 9f770464..970762f6 100644 --- a/services/follows/get_followers.py +++ b/services/follows/get_followers.py @@ -4,13 +4,13 @@ import follows_pb2 -class SendFollowServicer: +class GetFollowsReceiver: def __init__(self, logger, util, database_stub): self._logger = logger self._util = util self._database_stub = database_stub - self.RequestType = Enum('FOLLOWING', 'FOLLOWER') + self.RequestType = Enum('RequestType', 'FOLLOWING FOLLOWERS') def _get_follows(self, request, context, request_type): if request_type == self.RequestType.FOLLOWERS: @@ -19,7 +19,7 @@ def _get_follows(self, request, context, request_type): else: self._logger.debug('List of users %s is following requested', request.username) - resp = follows_pb2.GetFollowsReponse() + resp = follows_pb2.GetFollowsResponse() # Parse input username handle, host = self._util.parse_username( @@ -43,27 +43,34 @@ def _get_follows(self, request, context, request_type): # Get followers/followings for this user. following_ids = None if request_type == self.RequestType.FOLLOWERS: - following_ids = util.get_follows(followed_id=user_id) + following_ids = self._util.get_follows(followed_id=user_id).results else: - following_ids = util.get_follows(follower_id=user_id) + following_ids = self._util.get_follows(follower_id=user_id).results # Convert other following users and add to output proto. - for _id in following_ids: + for following_id in following_ids: + _id = following_id.follower + if request_type == self.RequestType.FOLLOWERS: + _id = following_id.followed user = self._util.get_user_from_db(global_id=_id) if user is None: self._logger.warning('Could not find user for id %d', _id) continue - if self._util.convert_db_user_to_follow_user(user, - resp.results.add()): - self._logger.warning('Could not conver user %s@%s to ' + + + ok = self._util.convert_db_user_to_follow_user(user, + resp.results.add()) + if not ok: + self._logger.warning('Could not convert user %s@%s to ' + 'FollowUser', user.handle, user.host) resp.result_type = follows_pb2.GetFollowsResponse.OK return resp def GetFollowing(self, request, context): + self._logger.debug('GetFollowing, username = %s', request.username) return self._get_follows(request, context, self.RequestType.FOLLOWING) def GetFollowers(self, request, context): - return self._get_follows(request, context, self.RequestType.FOLLOWER) + self._logger.debug('GetFollowers, username = %s', request.username) + return self._get_follows(request, context, self.RequestType.FOLLOWERS) From cfce1066462c77f12bab95194348820a18a596a0 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 22:23:36 +0000 Subject: [PATCH 14/18] Add missing field no. in proto --- services/follows/follows.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/follows/follows.proto b/services/follows/follows.proto index 7e0c030d..3d2fd030 100644 --- a/services/follows/follows.proto +++ b/services/follows/follows.proto @@ -53,7 +53,7 @@ message FollowUser { message GetFollowsRequest { /* Eg. "admin", "weeura@samsoniuk.ua", "@jose" */ - string username; + string username = 1; } message GetFollowsResponse { @@ -69,7 +69,7 @@ message GetFollowsResponse { string error = 2; /* Should only be set if result_type is OK. */ - repeated FollowUser users = 3; + repeated FollowUser results = 3; } service Follows { From fd98796e6779aba40997ec62cd26d1aa93f3a537 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 22:35:51 +0000 Subject: [PATCH 15/18] return correct error types --- services/database/follow_servicer.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/services/database/follow_servicer.py b/services/database/follow_servicer.py index fbd37239..257e7ed8 100644 --- a/services/database/follow_servicer.py +++ b/services/database/follow_servicer.py @@ -57,16 +57,21 @@ def _follow_handle_find(self, req, resp): filter_clause, values = util.entry_to_filter(req.match) try: if not filter_clause: - res = self._db.execute('SELECT * FROM follows') + query = 'SELECT * FROM follows' + self._logger.debug('Running query "%s"', query) + res = self._db.execute(query) else: - res = self._db.execute( - 'SELECT * FROM posts WHERE ' + filter_clause, - *values) + query = 'SELECT * FROM follows WHERE ' + filter_clause + valstr = ', '.join(str(v) for v in values) + self._logger.debug('Running query "%s" with values (%s)', + query, valstr) + res = self._db.execute(query, *values) except sqlite3.Error as e: - resp.result_type = database_pb2.DbFollowsResponse.ERROR + self._logger.warning('Got error reading DB: ' + str(e)) + resp.result_type = database_pb2.DbFollowResponse.ERROR resp.error = str(e) return - resp.result_type = database_pb2.DbFollowsResponse.OK + resp.result_type = database_pb2.DbFollowResponse.OK for tup in res: if not self._db_tuple_to_entry(tup, resp.results.add()): del resp.results[-1] From 7655289dc0d55a1dbcabb7f5c6ec7c9c74b665b6 Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 22:38:09 +0000 Subject: [PATCH 16/18] pep8, fix module name --- services/follows/servicer.py | 2 +- services/follows/util.py | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/services/follows/servicer.py b/services/follows/servicer.py index ac64833c..4b8fcc94 100644 --- a/services/follows/servicer.py +++ b/services/follows/servicer.py @@ -1,4 +1,4 @@ -from get_followers import GetFollowersReceiver +from get_followers import GetFollowsReceiver from receive_follow import ReceiveFollowServicer from send_follow import SendFollowServicer diff --git a/services/follows/util.py b/services/follows/util.py index d351c9fc..fee6ff6f 100644 --- a/services/follows/util.py +++ b/services/follows/util.py @@ -35,10 +35,10 @@ def _create_user_in_db(self, entry): return insert_resp def get_user_from_db(self, - handle=None, - host=None, - global_id=None, - attempt_number=0): + handle=None, + host=None, + global_id=None, + attempt_number=0): if attempt_number > MAX_FIND_RETRIES: self._logger.error('Retried query too many times.') return None @@ -94,26 +94,27 @@ def get_follows(self, follower_id=None, followed_id=None): self._logger.debug('Finding follows ', ('*' if (follower_id is None) else str(follower_id)), ('*' if (followed_id is None) else str(followed_id))) - follow_entry=database_pb2.Follow( + follow_entry = database_pb2.Follow( follower=follower_id, followed=followed_id ) - follow_req=database_pb2.DbFollowRequest( + follow_req = database_pb2.DbFollowRequest( request_type=database_pb2.DbFollowRequest.FIND, match=follow_entry ) - follow_resp=self._db.Follow(follow_req) + follow_resp = self._db.Follow(follow_req) if follow_resp.result_type == database_pb2.DbFollowResponse.ERROR: self._logger.error('Could not add follow to database: %s', follow_resp.error) return follow_resp def convert_db_user_to_follow_user(self, db_user, follow_user): - self._logger.warning('Trying to convert %s %s', db_user.handle, db_user.host) + self._logger.warning('Trying to convert %s %s', + db_user.handle, db_user.host) try: - follow_user.handle=db_user.handle - follow_user.host=db_user.host - follow_user.display_name=db_user.display_name + follow_user.handle = db_user.handle + follow_user.host = db_user.host + follow_user.display_name = db_user.display_name except Exception as e: self._logger.warning('Error converting db user to follow user: ' + str(e)) From 19b3e117246ee1412256d5ce9eab3ba75fd8daad Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 22:51:20 +0000 Subject: [PATCH 17/18] Swap follower and followed in response --- services/follows/get_followers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/follows/get_followers.py b/services/follows/get_followers.py index 970762f6..a7a15a8a 100644 --- a/services/follows/get_followers.py +++ b/services/follows/get_followers.py @@ -49,9 +49,9 @@ def _get_follows(self, request, context, request_type): # Convert other following users and add to output proto. for following_id in following_ids: - _id = following_id.follower + _id = following_id.followed if request_type == self.RequestType.FOLLOWERS: - _id = following_id.followed + _id = following_id.follower user = self._util.get_user_from_db(global_id=_id) if user is None: self._logger.warning('Could not find user for id %d', From d497d744f042d95d85cab02466d8bc933f3efadb Mon Sep 17 00:00:00 2001 From: iandioch Date: Sun, 28 Oct 2018 22:51:32 +0000 Subject: [PATCH 18/18] Add extra logging for debugging --- services/database/follow_servicer.py | 2 ++ services/follows/util.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/services/database/follow_servicer.py b/services/database/follow_servicer.py index 257e7ed8..65168c9e 100644 --- a/services/database/follow_servicer.py +++ b/services/database/follow_servicer.py @@ -75,3 +75,5 @@ def _follow_handle_find(self, req, resp): for tup in res: if not self._db_tuple_to_entry(tup, resp.results.add()): del resp.results[-1] + + self._logger.debug('%d results of follower query.', len(resp.results)) diff --git a/services/follows/util.py b/services/follows/util.py index fee6ff6f..5702e03b 100644 --- a/services/follows/util.py +++ b/services/follows/util.py @@ -109,7 +109,7 @@ def get_follows(self, follower_id=None, followed_id=None): return follow_resp def convert_db_user_to_follow_user(self, db_user, follow_user): - self._logger.warning('Trying to convert %s %s', + self._logger.warning('Converting db user %s@%s to follow user.', db_user.handle, db_user.host) try: follow_user.handle = db_user.handle