Sync GEO data with IFRC-GO

.dockerignore

@@ -0,0 +1,134 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+.azure
+
+# editors
+.idea/
+
+# In app data
+apps/cap_feed/geographical/

.github/pull_request_template.md

@@ -0,0 +1,28 @@
+Addresses
+- #example-issue-number
+
+Depends on
+- #example-Issue/PR-number
+
+NOTE: **Mention related users here if any.**
+
+## Changes
+
+* Detailed list or prose of changes
+* Breaking changes
+* Changes to configurations
+
+## This PR doesn't introduce any:
+
+- [ ] temporary files, auto-generated files or secret keys
+- [ ] n+1 queries
+- [ ] flake8 issues
+- [ ] `print`
+- [ ] typos
+- [ ] unwanted comments
+
+## This PR contains valid:
+
+- [ ] tests
+- [ ] permission checks (tests here too)
+- [ ] translations

.pre-commit-config.yaml

@@ -14,7 +14,7 @@ exclude: |
 
 repos:
   - repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.4.2
     hooks:
       - id: black
         # args: ["--check"]

Dockerfile

@@ -11,7 +11,7 @@ COPY pyproject.toml poetry.lock /code/
 RUN apt-get update -y \
     && apt-get install -y --no-install-recommends \
         # Build required packages
-        gcc libc-dev libproj-dev \
+        gcc libc-dev gdal-bin libproj-dev \
         # Helper packages
         wait-for-it \
     # Upgrade pip and install python packages for code

apps/cap_feed/admin.py

@@ -1,3 +1,4 @@
+from admin_auto_filters.filters import AutocompleteFilterFactory
 from django.contrib import admin
 from django_celery_beat.models import (
     ClockedSchedule,
@@ -44,7 +45,10 @@ class AlertInfoAreaAdmin(admin.ModelAdmin):
 
 class AlertInfoAdmin(admin.ModelAdmin):
     list_display = ['alert', 'language']
-    list_filter = ['alert__feed', 'alert__country']
+    list_filter = (
+        AutocompleteFilterFactory('Feed', 'alert__feed'),
+        AutocompleteFilterFactory('Country', 'alert__country'),
+    )
     search_fields = ['alert__url']
     fieldsets = [
         ('Administration', {'fields': ['alert']}),
@@ -83,7 +87,11 @@ class AlertInfoInline(admin.StackedInline):
 
 class AlertAdmin(admin.ModelAdmin):
     list_display = ['url', 'country', 'feed', 'sent', 'status', 'msg_type', 'scope']
-    list_filter = ['feed', 'country']
+    list_filter = (
+        'is_expired',
+        AutocompleteFilterFactory('Feed', 'feed'),
+        AutocompleteFilterFactory('Country', 'country'),
+    )
     search_fields = ['url']
     fieldsets = [
         ('Administration', {'fields': ['country', 'feed']}),
@@ -113,13 +121,16 @@ class AlertAdmin(admin.ModelAdmin):
 
 class CountryAdmin(admin.ModelAdmin):
     list_display = ['name', 'iso3', 'region', 'continent']
-    list_filter = ['region', 'continent']
+    list_filter = (
+        'region',
+        'continent',
+    )
     search_fields = ['name', 'iso3']
 
 
 class Admin1Admin(admin.ModelAdmin):
     list_display = ['name', 'country']
-    list_filter = ['country']
+    list_filter = (AutocompleteFilterFactory('Country', 'country'),)
     search_fields = ['name']
 
 
@@ -135,7 +146,11 @@ def get_formset(self, *args, **kwargs):
 
 class FeedAdmin(admin.ModelAdmin):
     list_display = ['name', 'country', 'url', 'format', 'polling_interval']
-    list_filter = ['format', 'country__region', 'country']
+    list_filter = (
+        'format',
+        'country__region',
+        AutocompleteFilterFactory('Country', 'country'),
+    )
     search_fields = ['url']
     inlines = [LanguageInfoInline]
 
@@ -153,7 +168,10 @@ def name(self, obj):
 
 class FeedLogAdmin(admin.ModelAdmin):
     list_display = ['exception', 'feed', 'description', 'alert_url', 'timestamp']
-    list_filter = ['exception', 'feed']
+    list_filter = (
+        'exception',
+        AutocompleteFilterFactory('Feed', 'feed'),
+    )
     search_fields = ['feed', 'exception', 'alert_url']
     fieldsets = [
         ('Log Context', {'fields': ['feed', 'alert_url', 'timestamp', 'notes']}),
@@ -163,13 +181,16 @@ class FeedLogAdmin(admin.ModelAdmin):
 
 class AlertAdmin1Admin(admin.ModelAdmin):
     list_display = ['alert', 'admin1']
-    list_filter = ['alert__country', 'admin1']
+    list_filter = (
+        AutocompleteFilterFactory('Admin1', 'admin1'),
+        AutocompleteFilterFactory('Country', 'alert__country'),
+    )
     search_fields = ['alert__url', 'admin1__name']
 
 
 class ProcessedAlertAdmin(admin.ModelAdmin):
     list_display = ['url', 'feed']
-    list_filter = ['feed']
+    list_filter = (AutocompleteFilterFactory('Feed', 'feed'),)
     search_fields = ['url']
 
 

apps/cap_feed/apps.py

@@ -1,5 +1,6 @@
 from django.apps import AppConfig
-from django.db.models.signals import post_delete, post_save, pre_delete
+
+# from django.db.models.signals import post_save, pre_delete
 
 
 class CapFeedConfig(AppConfig):
@@ -8,19 +9,16 @@ class CapFeedConfig(AppConfig):
 
     # Listen to the new registration event of feed
     def ready(self):
-        Feed = self.get_model("Feed")
-        Alert = self.get_model("Alert")
-        post_save.connect(notify_incoming_alert_for_subscription, sender=Alert)
-        post_delete.connect(notify_removed_alert_for_subscription, sender=Alert)
-        post_delete.connect(delete_feed, sender=Feed)
-        post_save.connect(update_cache_instructions, sender=Alert)
-        pre_delete.connect(update_cache_instructions, sender=Alert)
-
-
-def delete_feed(sender, instance, *args, **kwargs):
-    from .models import remove_task
+        # Feed = self.get_model("Feed")
+        # Alert = self.get_model("Alert")
+        # post_save.connect(notify_incoming_alert_for_subscription, sender=Alert)
+        # post_delete.connect(notify_removed_alert_for_subscription, sender=Alert)
+        # TODO: Clean-up cache
+        # post_save.connect(update_cache_instructions, sender=Alert)
+        # pre_delete.connect(update_cache_instructions, sender=Alert)
+        import apps.cap_feed.receivers
 
-    remove_task(instance)
+        apps.cap_feed.receivers
 
 
 def notify_incoming_alert_for_subscription(sender, instance, *args, **kwargs):
@@ -50,6 +48,7 @@ def notify_removed_alert_for_subscription(sender, instance, *args, **kwargs):
     )
 
 
+# TODO: Rmove this
 def update_cache_instructions(sender, instance, *args, **kwargs):
     from main.celery import app
 

apps/cap_feed/data_injector/feed.py

@@ -1,9 +1,11 @@
 import json
+import logging
 import os
 
 from apps.cap_feed.models import Country, Feed, LanguageInfo
 
 module_dir = os.path.dirname(__file__)  # get current directory
+logger = logging.getLogger(__name__)
 
 
 # inject feed configurations if not already present
@@ -14,7 +16,7 @@ def inject_feeds():
     )
     with open(file_path, encoding='utf-8') as file:
         feeds = json.load(file)
-        print('Injecting feeds...')
+        logger.info('Injecting feeds...')
         unique_countries = set()
         feed_counter = 0
         for feed_entry in feeds:
@@ -27,7 +29,7 @@ def inject_feeds():
                 if Feed.objects.filter(url=feed.url).first():
                     continue
                 feed.format = feed_entry['format']
-                feed.polling_interval = 60
+                feed.polling_interval = Feed.PoolingInterval.I_10m
                 feed.enable_polling = True
                 feed.enable_rebroadcast = True
                 feed.official = True
@@ -40,8 +42,7 @@ def inject_feeds():
                 language_info.logo = feed_entry['picUrl']
                 language_info.save()
 
-            except Exception as e:
-                print(feed_entry['name'])
-                print(f'Error injecting feed: {e}')
+            except Exception:
+                logger.error(f"Error injecting feed: {feed_entry['name']}", exc_info=True)
 
-        print(f'Injected {feed_counter} feeds for {len(unique_countries)} unique countries')
+        logger.info(f'Injected {feed_counter} feeds for {len(unique_countries)} unique countries')