Update graylog logging configuration

We moved graylog to kubernetes, which means that we need to use https as the transport to send messages to a k8s ingress. graypy doesn't support https, only http, so switch to pygelf which does support it.

requirements.in

@@ -28,7 +28,6 @@ fabric==2.6.0
 feedparser~=6.0.10
 freezegun==1.2.2
 future~=0.18.2
-graypy==0.2.12
 gunicorn==21.2.0
 ipython==8.14.0
 jinja2==3.0.3  # This version needed for sphinx to not raise errors
@@ -42,6 +41,7 @@ openpyxl==3.1.0  # for reading .xlsx files (but not .xls)
 Pillow==9.5.0
 pip-tools==7.1.0
 psycopg2-binary==2.9.6
+pygelf==0.4.2
 PyJWT==2.6.0
 pyparsing==2.4.7
 pysndfile==1.4.4

requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
-#    pip-compile requirements.in
+#    pip-compile
 #
 akismet==1.0.1
     # via -r requirements.in
@@ -150,8 +150,6 @@ future==0.18.3
     # via -r requirements.in
 gprof2dot==2022.7.29
     # via django-silk
-graypy==0.2.12
-    # via -r requirements.in
 gunicorn==21.2.0
     # via -r requirements.in
 idna==3.6
@@ -245,6 +243,8 @@ pycodestyle==2.11.1
     # via autopep8
 pycparser==2.21
     # via cffi
+pygelf==0.4.2
+    # via -r requirements.in
 pygments==2.17.2
     # via
     #   ipython

similarity/requirements.txt

@@ -1,6 +1,6 @@
 PyYAML==5.3
 Twisted==20.3.0
-graypy==2.1.0
+pygelf==0.4.2
 concurrent-log-handler==0.9.20
 #gaia2
 future

similarity/similarity_indexing_server.py

@@ -32,7 +32,7 @@
 import logging
 from logging.handlers import RotatingFileHandler
 
-import graypy
+import pygelf
 from twisted.internet import reactor
 from twisted.web import server, resource
 
@@ -95,8 +95,8 @@ def clear_memory(self, request):
     logger.addHandler(handler)
     std_handler.setFormatter(formatter)
     logger.addHandler(std_handler)
-    handler_graypy = graypy.GELFHandler('10.55.0.48', 12201)
-    logger.addHandler(handler_graypy)
+    handler_gelf = pygelf.GelfHttpsHandler(sim_settings.LOGSERVER_HOST, sim_settings.LOGSERVER_PORT)
+    logger.addHandler(handler_gelf)
 
     # Start service
     logger.info('Configuring similarity INDEXING service...')

similarity/similarity_server.py

@@ -25,13 +25,13 @@
 import json
 import logging
 
-import graypy
+import pygelf
 import yaml
 from concurrent_log_handler import ConcurrentRotatingFileHandler
 from gaia_wrapper import GaiaWrapper
 from similarity_server_utils import parse_filter, parse_target, parse_metric_descriptors
 from similarity_settings import LISTEN_PORT, LOGFILE, DEFAULT_PRESET, DEFAULT_NUMBER_OF_RESULTS, INDEX_NAME, \
-    BAD_REQUEST_CODE, LOGSERVER_IP_ADDRESS, LOGSERVER_PORT, LOG_TO_STDOUT, \
+    BAD_REQUEST_CODE, LOGSERVER_HOST, LOGSERVER_PORT, LOG_TO_STDOUT, \
     LOG_TO_GRAYLOG, LOG_TO_FILE
 from twisted.internet import reactor
 from twisted.web import server, resource
@@ -248,8 +248,8 @@ def save(self, request, filename=None):
         std_handler.setFormatter(formatter)
         logger.addHandler(std_handler)
     if LOG_TO_GRAYLOG:
-        handler_graypy = graypy.GELFHandler(LOGSERVER_IP_ADDRESS, LOGSERVER_PORT)
-        logger.addHandler(handler_graypy)
+        handler_gelf = pygelf.GelfHttpsHandler(LOGSERVER_HOST, LOGSERVER_PORT)
+        logger.addHandler(handler_gelf)
 
     # Start service
     logger.info('Configuring similarity service...')

similarity/similarity_settings.py

@@ -53,7 +53,7 @@
 NOT_FOUND_CODE = 404
 
 # Graylog GELF endpoint
-LOGSERVER_IP_ADDRESS = 'IP_ADDRESS'
+LOGSERVER_HOST = 'IP_ADDRESS'
 LOGSERVER_PORT = 0000
 
 # Set to true to log to stdout in addition to files and graylog

tagrecommendation/requirements.txt

@@ -1,5 +1,5 @@
 Twisted==20.3.0
-graypy==2.1.0
+pygelf==0.4.2
 concurrent-log-handler==0.9.20
 scipy==1.2.3
 numpy==1.9.0  # Needs to be this version for csc-pysparse to work

tagrecommendation/tagrecommendation_server.py

@@ -30,7 +30,7 @@
 import json
 import logging
 
-import graypy
+import pygelf
 import tagrecommendation_settings as tr_settings
 from tag_recommendation.community_tag_recommender import CommunityBasedTagRecommender
 from concurrent_log_handler import ConcurrentRotatingFileHandler
@@ -172,8 +172,8 @@ def add_to_index(self, sound_ids, sound_tagss):
         std_handler.setFormatter(formatter)
         logger.addHandler(std_handler)
     if tr_settings.LOG_TO_GRAYLOG:
-        handler_graypy = graypy.GELFHandler(tr_settings.LOGSERVER_IP_ADDRESS, tr_settings.LOGSERVER_PORT)
-        logger.addHandler(handler_graypy)
+        handler_gelf = pygelf.GelfHttpsHandler(tr_settings.LOGSERVER_HOST, tr_settings.LOGSERVER_PORT)
+        logger.addHandler(handler_gelf)
 
     # Start service
     logger.info('Configuring tag recommendation service...')

tagrecommendation/tagrecommendation_settings.py

@@ -28,7 +28,7 @@
 RECOMMENDATION_TMP_DATA_DIR = os.path.join(RECOMMENDATION_DATA_DIR, 'tmp')
 
 # Graylog GELF endpoint
-LOGSERVER_IP_ADDRESS = 'IP_ADDRESS'
+LOGSERVER_HOST = 'IP_ADDRESS'
 LOGSERVER_PORT = 0000
 
 # Set to true to log to stdout in addition to files and graylog