diff --git a/ckanext-hdx_crisis/ckanext/hdx_crisis/controllers/custom_location_controller.py b/ckanext-hdx_crisis/ckanext/hdx_crisis/controllers/custom_location_controller.py index 53f07cd233..6f55706fb4 100644 --- a/ckanext-hdx_crisis/ckanext/hdx_crisis/controllers/custom_location_controller.py +++ b/ckanext-hdx_crisis/ckanext/hdx_crisis/controllers/custom_location_controller.py @@ -96,6 +96,7 @@ def generate_template_data(self, id, group_info, custom_dict): template_data = { 'data': { + 'country_id': group_info['id'], 'country_name': group_info['name'], 'country_title': group_info.get('title', group_info['name']), 'topline_chart_sections': self._create_sections(top_line_items, charts_config_data), diff --git a/ckanext-hdx_org_group/ckanext/hdx_org_group/controllers/organization_controller.py b/ckanext-hdx_org_group/ckanext/hdx_org_group/controllers/organization_controller.py index 6adc2b5477..dbf463d484 100644 --- a/ckanext-hdx_org_group/ckanext/hdx_org_group/controllers/organization_controller.py +++ b/ckanext-hdx_org_group/ckanext/hdx_org_group/controllers/organization_controller.py @@ -273,7 +273,9 @@ def edit(self, id, data=None, errors=None, error_summary=None): self._setup_template_variables(context, data, group_type=group_type) c.form = render(self._group_form(group_type), extra_vars=vars) - return render(self._edit_template(c.group.type)) + + # The extra_vars are needed here to send analytics information like org name and id + return render(self._edit_template(c.group.type), extra_vars={'data': data}) def check_access(self, action_name, data_dict=None): if data_dict is None: diff --git a/ckanext-hdx_package/ckanext/hdx_package/controllers/contribute_flow_controller.py b/ckanext-hdx_package/ckanext/hdx_package/controllers/contribute_flow_controller.py index e6d27a5e37..da1cd7a623 100644 --- a/ckanext-hdx_package/ckanext/hdx_package/controllers/contribute_flow_controller.py +++ b/ckanext-hdx_package/ckanext/hdx_package/controllers/contribute_flow_controller.py @@ -7,6 +7,8 @@ import ckan.lib.navl.dictization_functions as dict_fns import ckan.lib.helpers as h +import ckanext.hdx_package.helpers.analytics as analytics + from ckan.common import _, request, response, c from ckan.lib.search import SearchIndexError from ckan.controllers.api import CONTENT_TYPES @@ -110,8 +112,12 @@ def _abort(self, save_type, status_code, message): def _prepare_and_render(self, save_type='', data=None, errors=None, error_summary=None): save_type = save_type if save_type else '' + + analytics_dict = self._generate_analytics_data(data) + template_data = { 'data': data, + 'analytics': analytics_dict, 'errors': errors, 'error_summary': error_summary, 'aborted': False @@ -123,6 +129,20 @@ def _prepare_and_render(self, save_type='', data=None, errors=None, error_summar else: return base.render('contribute_flow/create_edit.html', extra_vars=template_data) + def _generate_analytics_data(self, data): + # in case of an edit event we populate the analytics info + analytics_dict = {} + if data and data.get('id'): + analytics_dict['is_cod'] = analytics.is_cod(data) + analytics_dict['is_indicator'] = analytics.is_indicator(data) + analytics_dict['group_names'], analytics_dict['group_ids'] = analytics.extract_locations_in_json(data) + else: + analytics_dict['is_cod'] = 'false' + analytics_dict['is_indicator'] = 'false' + analytics_dict['group_names'] = '[]' + analytics_dict['group_ids'] = '[]' + return analytics_dict + def _save_or_update(self, context, package_type=None): data_dict = {} try: diff --git a/ckanext-hdx_package/ckanext/hdx_package/controllers/dataset_controller.py b/ckanext-hdx_package/ckanext/hdx_package/controllers/dataset_controller.py index 6440df67bd..a493ef96f8 100644 --- a/ckanext-hdx_package/ckanext/hdx_package/controllers/dataset_controller.py +++ b/ckanext-hdx_package/ckanext/hdx_package/controllers/dataset_controller.py @@ -31,6 +31,8 @@ import ckan.lib.dictization.model_dictize as model_dictize import ckan.lib.search as search +import ckanext.hdx_package.helpers.analytics as analytics + from ckan.common import _, json, request, c, g, response from ckan.controllers.home import CACHE_PARAMETERS @@ -682,6 +684,12 @@ def read(self, id, format='html'): context, {'id': resource['id']}) resource['has_views'] = len(resource_views) > 0 + if helpers.is_ckan_domain(resource['url']): + resource['url'] = helpers.make_url_relative(resource['url']) + + if resource.get('perma_link') and helpers.is_ckan_domain(resource['perma_link']): + resource['perma_link'] = helpers.make_url_relative(resource['perma_link']) + # Is this an indicator? Load up graph data #c.pkg_dict['indicator'] = 1 try: @@ -704,8 +712,9 @@ def read(self, id, format='html'): template = template[:template.index('.') + 1] + format # set dataset type for google analytics - modified by HDX - c.ga_dataset_type = self._google_analytics_dataset_type(c.pkg_dict) - c.ga_location = self._google_analytics_location(c.pkg_dict) + c.analytics_is_cod = analytics.is_cod(c.pkg_dict) + c.analytics_is_indicator = analytics.is_indicator(c.pkg_dict) + c.analytics_group_names, c.analytics_group_ids = analytics.extract_locations_in_json(c.pkg_dict) # changes done for indicator act_data_dict = {'id': c.pkg_dict['id'], 'limit': 7} @@ -770,32 +779,6 @@ def read(self, id, format='html'): assert False, "We should never get here" - def _google_analytics_dataset_type(self, pkg_dict): - type = 'standard' - tags = [tag.get('name', '') for tag in pkg_dict.get('tags', [])] - - if int(pkg_dict.get('indicator', 0)) == 1: - type = 'indicator' - if 'cod' in tags: - type = 'cod~indicator' if type == 'indicator' else 'cod' - - return type - - def _google_analytics_location(self, pkg_dict): - limit = 15 - locations = pkg_dict.get('groups', []) - if len(locations) >= limit: - result = 'many' - else: - locations = [item.get('name', '') for item in locations] - locations.sort() - result = "~".join(locations) - - if not result: - result = 'none' - - return result - def _get_org_extras(self, org_id): """ Get the extras for our orgs @@ -1101,8 +1084,10 @@ def resource_read(self, id, resource_id): c.resource['has_views'] = len(resource_views) > 0 # set dataset type for google analytics - modified by HDX - c.ga_dataset_type = self._google_analytics_dataset_type(c.package) - c.ga_location = self._google_analytics_location(c.package) + # c.ga_dataset_type = self._google_analytics_dataset_type(c.package) + c.analytics_is_cod = analytics.is_cod(c.package) + c.analytics_is_indicator = analytics.is_indicator(c.package) + c.analytics_group_names, c.analytics_group_ids = analytics.extract_locations_in_json(c.package) current_resource_view = None view_id = request.GET.get('view_id') @@ -1123,6 +1108,13 @@ def resource_read(self, id, resource_id): 'current_resource_view': current_resource_view, 'dataset_type': dataset_type} + download_url = c.resource.get('perma_link') if c.resource.get('perma_link') else c.resource['url'] + c.resource['original_url'] = download_url + c.resource['download_url'] = download_url + if helpers.is_ckan_domain(download_url): + c.resource['download_url'] = helpers.make_url_relative(download_url) + + template = self._resource_template(dataset_type) return render(template, extra_vars=vars) diff --git a/ckanext-hdx_package/ckanext/hdx_package/helpers/analytics.py b/ckanext-hdx_package/ckanext/hdx_package/helpers/analytics.py new file mode 100644 index 0000000000..15e0f4ce75 --- /dev/null +++ b/ckanext-hdx_package/ckanext/hdx_package/helpers/analytics.py @@ -0,0 +1,182 @@ +import logging +import json +import urlparse +import requests + +import pylons.config as config + +import ckan.model as model +import ckan.lib.base as base +import ckan.logic as logic +import ckan.controllers.package as package_controller + +from ckan.common import _, c, request + +log = logging.getLogger(__name__) + + +def is_indicator(pkg_dict): + if int(pkg_dict.get('indicator', 0)) == 1: + return 'true' + return 'false' + + +def is_cod(pkg_dict): + tags = [tag.get('name', '') for tag in pkg_dict.get('tags', [])] + if 'cod' in tags: + return 'true' + return 'false' + + +def extract_locations(pkg_dict): + locations = pkg_dict.get('groups', []) + location_names = [] + location_ids = [] + for l in sorted(locations, key=lambda item: item.get('name', '')): + location_names.append(l.get('name', '')) + location_ids.append(l.get('id', '')) + + return location_names, location_ids + + +def extract_locations_in_json(pkg_dict): + locations = pkg_dict.get('groups', []) + location_names = [] + location_ids = [] + for l in sorted(locations, key=lambda item: item.get('name', '')): + location_names.append(l.get('name', '')) + location_ids.append(l.get('id', '')) + + return json.dumps(location_names), json.dumps(location_ids) + + +def _ga_dataset_type(is_indicator, is_cod): + ''' + :param is_indicator: + :type is_indicator: bool + :param is_cod: + :type is_cod: bool + :return: standard / indicator / cod / cod~indicator + :rtype: str + ''' + + type = 'standard' + if is_indicator: + type = 'indicator' + if is_cod: + type = 'cod~indicator' if type == 'indicator' else 'cod' + + return type + + +def _ga_location(location_names): + ''' + :param location_names: + :type location_names: list[str] + :return: + :rtype: str + ''' + limit = 15 + if len(location_names) >= limit: + result = 'many' + else: + result = "~".join(location_names) + + if not result: + result = 'none' + + return result + + +def wrap_resource_download_function(): + original_resource_download = package_controller.PackageController.resource_download + + def new_resource_download(self, id, resource_id, filename=None): + send_event = True + + referer_url = request.referer + remote_addr = request.remote_addr + request_url = request.url + + if referer_url: + ckan_url = config.get('ckan.site_url', '//localhost:5000') + ckan_parsed_url = urlparse.urlparse(ckan_url) + referer_parsed_url = urlparse.urlparse(referer_url) + + if ckan_parsed_url.hostname == referer_parsed_url.hostname: + send_event = False + try: + if send_event: + context = {'model': model, 'session': model.Session, + 'user': c.user or c.author, 'auth_user_obj': c.userobj} + resource_dict = logic.get_action('resource_show')(context, {'id': resource_id}) + dataset_dict = logic.get_action('package_show')(context, {'id': id}) + location_names, location_ids = extract_locations(dataset_dict) + + dataset_title = dataset_dict.get('title', dataset_dict.get('name')) + dataset_is_cod = is_cod(dataset_dict) == 'true' + dataset_is_indicator = is_indicator(dataset_dict) == 'true' + + analytics_enqueue_url = config.get('hdx.analytics.enqueue_url') + analytics_dict = { + 'event_name': 'resource download', + 'mixpanel_tracking_id': 'anonymous', + 'mixpanel_token': config.get('hdx.analytics.mixpanel.token'), + 'send_mixpanel': True, + 'send_ga': True, + 'mixpanel_meta': { + "resource name": resource_dict.get('name'), + "resource id": resource_dict.get('id'), + "dataset name": dataset_dict.get('title'), + "dataset id": dataset_dict.get('id'), + "org name": dataset_dict.get('organization', {}).get('name'), + "org id": dataset_dict.get('organization', {}).get('id'), + "group names": location_names, + "group ids": location_ids, + "is cod": dataset_is_cod, + "is indicator": dataset_is_indicator, + "event source": "direct", + "referer url": referer_url + }, + 'ga_meta': { + 'v': '1', + 't': 'event', + 'cid': 'anonymous', + 'tid': config.get('hdx.analytics.ga.token'), + 'ds': 'direct', + 'uip': remote_addr, + 'ec': 'resource', # event category + 'ea': 'download', # event action + 'dl': request_url, + 'el': '{} ({})'.format(resource_dict.get('name'), dataset_title), # event label + 'cd1': dataset_dict.get('organization', {}).get('name'), + 'cd2': _ga_dataset_type(dataset_is_indicator, dataset_is_cod), # type + 'cd3': _ga_location(location_names), # locations + + + + + } + } + + response = requests.post(analytics_enqueue_url, allow_redirects=True, timeout=2, + data=json.dumps(analytics_dict), headers={'Content-type': 'application/json'}) + response.raise_for_status() + enq_result = response.json() + log.info('Enqueuing result was: {}'.format(enq_result.get('success'))) + except logic.NotFound: + base.abort(404, _('Resource not found')) + except logic.NotAuthorized: + base.abort(401, _('Unauthorized to read resource %s') % id) + except requests.ConnectionError, e: + log.error("There was a connection error to the analytics enqueuing service: {}".format(str(e))) + except requests.HTTPError, e: + log.error("Bad HTTP response from analytics enqueuing service: {}".format(str(e))) + except requests.Timeout, e: + log.error("Request timed out: {}".format(str(e))) + except Exception, e: + log.error('Unexpected error {}'.format(e)) + + return original_resource_download(self, id, resource_id, filename) + + package_controller.PackageController.resource_download = new_resource_download diff --git a/ckanext-hdx_package/ckanext/hdx_package/helpers/helpers.py b/ckanext-hdx_package/ckanext/hdx_package/helpers/helpers.py index 5202d0550e..ecc49b1507 100644 --- a/ckanext-hdx_package/ckanext/hdx_package/helpers/helpers.py +++ b/ckanext-hdx_package/ckanext/hdx_package/helpers/helpers.py @@ -565,17 +565,13 @@ def hdx_get_proxified_resource_url(data_dict, proxy_schemes=['http','https']): 2) Return a domain relative url (without schema, domain or port) for local resources. :param data_dict: contains a resource and package dict - :type data_dict: dictionary + :type data_dict: dict :param proxy_schemes: list of url schemes to proxy for. :type data_dict: list ''' - ckan_url = config.get('ckan.site_url', '//localhost:5000') - url = data_dict['resource']['url'] - - parsed_url = urlparse.urlparse(url) - ckan_parsed_url = urlparse.urlparse(ckan_url) - same_domain = True if not parsed_url.hostname or parsed_url.hostname == ckan_parsed_url.hostname else False + same_domain = is_ckan_domain(data_dict['resource']['url']) + parsed_url = urlparse.urlparse(data_dict['resource']['url']) scheme = parsed_url.scheme if not same_domain and scheme in proxy_schemes: @@ -590,6 +586,30 @@ def hdx_get_proxified_resource_url(data_dict, proxy_schemes=['http','https']): return url +def is_ckan_domain(url): + ''' + :param url: url to check whether it's on the same domain as ckan + :type url: str + :return: True if it's the same domain. False otherwise + :rtype: bool + ''' + ckan_url = config.get('ckan.site_url', '//localhost:5000') + parsed_url = urlparse.urlparse(url) + ckan_parsed_url = urlparse.urlparse(ckan_url) + same_domain = True if not parsed_url.hostname or parsed_url.hostname == ckan_parsed_url.hostname else False + return same_domain + +def make_url_relative(url): + ''' + Transforms something like http://testdomain.com/test to /test + :param url: url to check whether it's on the same domain as ckan + :type url: str + :return: the new url as a string + :rtype: str + ''' + parsed_url = urlparse.urlparse(url) + return urlparse.urlunparse((None, None) + parsed_url[2:]) + def generate_mandatory_fields(): ''' diff --git a/ckanext-hdx_package/ckanext/hdx_package/plugin.py b/ckanext-hdx_package/ckanext/hdx_package/plugin.py index 927441d136..4befc847fc 100644 --- a/ckanext-hdx_package/ckanext/hdx_package/plugin.py +++ b/ckanext-hdx_package/ckanext/hdx_package/plugin.py @@ -31,6 +31,7 @@ import ckanext.hdx_package.actions.delete as hdx_delete import ckanext.hdx_package.helpers.helpers as hdx_helpers import ckanext.hdx_package.helpers.tracking_changes as tracking_changes +import ckanext.hdx_package.helpers.analytics as analytics import ckanext.hdx_package.actions.get as hdx_get import ckanext.hdx_org_group.helpers.organization_helper as org_helper @@ -56,6 +57,9 @@ def run_on_startup(): # replace original get_proxified_resource_url, check hdx_get_proxified_resource_url for more info resourceproxy_plugin.get_proxified_resource_url = hdx_helpers.hdx_get_proxified_resource_url + # wrap resource download function so that we can track download events + analytics.wrap_resource_download_function() + def _generate_license_list(): package.Package._license_register = license.LicenseRegister() diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/google-analytics.js b/ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/google-analytics.js index c5bc0875d6..510b720370 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/google-analytics.js +++ b/ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/google-analytics.js @@ -1,31 +1,148 @@ +$(function setUpSearchTracking() { + var formEl = $("#dataset-filter-form"); + if (formEl.length > 0) { + var mixpanelMapping = { + 'q': { + 'name': 'search term', + 'isList': false, + 'mandatory': true + }, + 'tags': { + 'name': 'tag filters', + 'isList': true, + 'mandatory': true + }, + 'res_format': { + 'name': 'format filters', + 'isList': true, + 'mandatory': true + }, + 'organization': { + 'name': 'org filters', + 'isList': true, + 'mandatory': true + }, + 'groups': { + 'name': 'location filters', + 'isList': true, + 'mandatory': true + }, + /*'ext_page_size': { + 'name': 'items per page', + 'isList': false, + 'mandatory': false + }, + 'sort': { + 'name': 'sorting', + 'isList': false, + 'mandatory': false + },*/ + 'ext_cod': { + 'name': 'cod filter', + 'isList': false, + 'mandatory': true + } + }; + var numberOfResults = parseInt($('#analytics-number-of-results').text().trim()) || 0; -function setUpResourcesTracking(){ - $('.ga-download').on('click', function(){ - //var rTitle = $(this).parents(".resource-item").find(".heading").attr("title"); - //var dTitle = $(".itemTitle").text().trim(); - var rTitle = $(this).find(".ga-download-resource-title").text().trim(); - var dTitle = $(this).find(".ga-download-dataset-title").text().trim(); - ga('send', 'event', 'resource', 'download', rTitle + " (" + dTitle +")"); - ga('send', 'event', 'dataset', 'resource-download', dTitle); - }); + var paramList = formEl.serializeArray(); + var mixpanelEventMeta = { + "page title": analyticsInfo.pageTitle, + "number of results": numberOfResults + /*"org name": analyticsInfo.organizationName, + "org id": analyticsInfo.organizationId, + "group names": analyticsInfo.groupNames, + "group ids": analyticsInfo.groupIds*/ + }; + var sendTrackingEvent = false; + for (var i = 0; i < paramList.length; i++) { + var param = paramList[i]; + var mappingInfo = mixpanelMapping[param.name]; + var paramValue = param.value.trim(); + if (mappingInfo && paramValue) { + populateMetadata(mixpanelEventMeta, mappingInfo, paramValue); + sendTrackingEvent = sendTrackingEvent || mappingInfo.mandatory; + } + } + if (sendTrackingEvent){ + var reResult = /ext_search_source=([^&]+)(&|$)/.exec(location.href); + if (reResult && reResult.length > 1) { + mixpanelEventMeta["search box location"] = reResult[1]; + } + else { + mixpanelEventMeta["search box location"] = "in-page"; + } + console.log(JSON.stringify(mixpanelEventMeta)); + mixpanel.track("search", mixpanelEventMeta); + } + else { + console.log("No mandatory properties found. Not sending search event to mixpanel."); + } + } - $('.ga-share').on('click', function(){ - var rTitle = $(this).parents(".resource-item").find(".heading").attr("title"); - var dTitle = $(".itemTitle").text().trim(); - ga('send', 'event', 'resource', 'share', rTitle + " (" + dTitle +")"); - ga('send', 'event', 'dataset', 'resource-share', dTitle); - }); + /** + * Populates the object that is sent to mixpanel for one
parameter + * @param mixpanelEventMeta {Object} map of property-values to be sent to mixpanel + * @param mappingInfo {{name:string, isList: boolean, mandatory: boolean}} information about how the param should be formatted + * @param paramValue {string} the value of the parameter + */ + function populateMetadata(mixpanelEventMeta, mappingInfo, paramValue) { + if (mappingInfo.isList) { + mixpanelEventMeta[mappingInfo.name] = mixpanelEventMeta[mappingInfo.name] ? + mixpanelEventMeta[mappingInfo.name] : []; + mixpanelEventMeta[mappingInfo.name].push(paramValue); + } + else { + mixpanelEventMeta[mappingInfo.name] = paramValue; + } + } +}); - $('.ga-preview').on('click', function(){ - console.log("sending event"); - var rTitle = $(this).parents(".resource-item").find(".heading").attr("title"); - var dTitle = $(".itemTitle").text().trim(); - ga('send', 'event', 'resource', 'preview', rTitle + " (" + dTitle +")"); - ga('send', 'event', 'dataset', 'resource-preview', dTitle); - }); -} +(function() { + function setUpResourcesTracking() { + $('.ga-download').on('click', function () { + var rTitle = $(this).find(".ga-download-resource-title").text().trim(); + var rId = $(this).find(".ga-download-resource-id").text().trim(); + // var dTitle = $(this).find(".ga-download-dataset-title").text().trim(); + var dTitle = analyticsInfo.datasetName; + ga('send', 'event', 'resource', 'download', rTitle + " (" + dTitle + ")"); + ga('send', 'event', 'dataset', 'resource-download', dTitle); + + mixpanel.track("resource download", { + "event source": "web", + "resource name": rTitle, + "resource id": rId, + "dataset name": analyticsInfo.datasetName, + "dataset id": analyticsInfo.datasetId, + "page title": analyticsInfo.pageTitle, + "org name": analyticsInfo.organizationName, + "org id": analyticsInfo.organizationId, + "group names": analyticsInfo.groupNames, + "group ids": analyticsInfo.groupIds, + "is cod": analyticsInfo.isCod, + "is indicator": analyticsInfo.isIndicator + }); + }); + + $('.ga-share').on('click', function () { + var rTitle = $(this).parents(".resource-item").find(".heading").attr("title"); + var dTitle = $(".itemTitle").text().trim(); + ga('send', 'event', 'resource', 'share', rTitle + " (" + dTitle + ")"); + ga('send', 'event', 'dataset', 'resource-share', dTitle); + }); + + $('.ga-preview').on('click', function () { + console.log("sending event"); + var rTitle = $(this).parents(".resource-item").find(".heading").attr("title"); + var dTitle = $(".itemTitle").text().trim(); + ga('send', 'event', 'resource', 'preview', rTitle + " (" + dTitle + ")"); + ga('send', 'event', 'dataset', 'resource-preview', dTitle); + }); + } + setUpResourcesTracking(); +}()); function setUpShareTracking(){ $(".indicator-actions.followButtonContainer a").on('click', function (){ var dTitle = $(".itemTitle").text().trim(); @@ -39,6 +156,4 @@ function setUpGalleryTracking() { var dTitle = $(".itemTitle").text().trim(); ga('send', 'event', 'gallery', 'click', rTitle + " (" + dTitle +")"); }); -} - -setUpResourcesTracking(); \ No newline at end of file +} \ No newline at end of file diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/indicator.js b/ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/indicator.js index a55a4d10aa..c24c01c054 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/indicator.js +++ b/ckanext-hdx_theme/ckanext/hdx_theme/fanstatic/indicator.js @@ -21,7 +21,6 @@ $(document).ready(function() { $("#location-dd").find("span").text($(this).text()); }); - //setUpResourcesTracking(); // directly called now from google-analytics.js setUpShareTracking(); setUpGalleryTracking(); }); diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/base.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/base.html index b78468cf07..41fc2a3e42 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/base.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/base.html @@ -10,6 +10,10 @@ {%- endblock -%} +{% block mixpanel_init %} + {{ h.snippet('snippets/mixpanel.html') }} +{% endblock %} + {# Allows custom attributes to be added to the tag #} @@ -20,29 +24,61 @@ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) })(window,document,'script','//www.google-analytics.com/analytics.js','ga'); - ga('create', 'UA-48221887-3', 'auto'); + var gaToken = '{{ h.hdx_get_ckan_config('hdx.analytics.ga.token') }}'; + ga('create', gaToken, 'auto'); //swap the two create calls in order to enable recording data from localhost instances //this is useful when testing to see realtime data on the Google Analytics site //ga('create', 'UA-48221887-3', { // 'cookieDomain': 'none' //}); - var dimension1Org = '{% block ga_dimension1 %}{% endblock %}'; - if ( dimension1Org ) { - ga('set', 'dimension1', dimension1Org); - } - - var dimension2Type = '{% block ga_dimension2 %}{% endblock %}'; - if ( dimension2Type ) { - ga('set', 'dimension2', dimension2Type); - } - - var dimension3Location = '{% block ga_dimension3 %}{% endblock %}'; - if ( dimension3Location ) { - ga('set', 'dimension3', dimension3Location); - } - ga('send', 'pageview'); + /** + * A map holding all the information to be sent to the analytics servers + * @type {object} + */ + var analyticsInfo = { + 'organizationName': '{% block analytics_org_name %}None{% endblock %}', + 'organizationId': '{% block analytics_org_id %}None{% endblock %}', + 'groupNames': {% block analytics_group_names %}[]{% endblock %}, + 'groupIds': {% block analytics_group_ids %}[]{% endblock %}, + 'isCod': {% block analytics_is_cod %}false{% endblock %}, + 'isIndicator': {% block analytics_is_indicator %}false{% endblock %}, + 'datasetName': '{% block analytics_dataset_name %}None{% endblock %}', + 'datasetId': '{% block analytics_dataset_id %}None{% endblock %}', + 'pageTitle': '{% if self.subtitle()|trim %}{{ self.subtitle()|trim }}{% else %} {{ 'None' }} {% endif -%}' + + }; + + (function() { + if (analyticsInfo.organizationName != 'None') { + ga('set', 'dimension1', analyticsInfo.organizationName); + } + + var dimension2Type = analyticsInfo.isCod ? 'cod' : ''; + dimension2Type += analyticsInfo.isCod && analyticsInfo.isIndicator ? '~' : ''; + dimension2Type += analyticsInfo.isIndicator ? 'indicator' : ''; + + if (dimension2Type) { + ga('set', 'dimension2', dimension2Type); + } + + var dimension3Location = analyticsInfo.groupNames.length < 15 ? analyticsInfo.groupNames.join('~') : 'many'; + if (dimension3Location) { + ga('set', 'dimension3', dimension3Location); + } + ga('send', 'pageview'); + + mixpanel.track("page view", { + "page title": analyticsInfo.pageTitle, + "org name": analyticsInfo.organizationName, + "org id": analyticsInfo.organizationId, + "group names": analyticsInfo.groupNames, + "group ids": analyticsInfo.groupIds, + "is cod": analyticsInfo.isCod, + "is indicator": analyticsInfo.isIndicator + }); + }()); @@ -60,6 +96,7 @@ #} {%- block meta -%} + {% block meta_generator %}{% endblock %} {%- endblock -%} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/contribute_flow/create_edit.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/contribute_flow/create_edit.html index 18f1a60357..ae0f346107 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/contribute_flow/create_edit.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/contribute_flow/create_edit.html @@ -2,10 +2,22 @@ {% import 'macros/form.html' as form %} +{# The line below is for google analytics #} +{% block analytics_org_name %}{{ data.get('organization',{}).name }}{% endblock %} +{% block analytics_org_id %}{{ data.get('organization',{}).id }}{% endblock %} +{% block analytics_is_cod %}{{ analytics.is_cod }}{% endblock %} +{% block analytics_is_indicator %}{{ analytics.is_indicator }}{% endblock %} +{% block analytics_group_names %}{{ analytics.group_names | safe }}{% endblock %} +{% block analytics_group_ids %}{{ analytics.group_ids | safe }}{% endblock %} +{% block analytics_dataset_name %}{{ data.name }}{% endblock %} +{% block analytics_dataset_id %}{{ data.id }}{% endblock %} + +{% block subtitle %}{{ _('Contribute') }}{% endblock %} + {% set edit_mode = True if data else False %} {% set data = data if data else h.generate_mandatory_fields() %} {% set errors = {} %} -{% set test_options = [{},{'value':'test', 'text':'Test'}, {'value':'test2', 'text':'Test2'}] %} + {% set user_is_sysadmin = h.check_access('sysadmin') %} {% set form_title = _('Edit Details') if edit_mode else _('Add Details') %} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/country.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/country.html index adfbe9613c..9d7a8b6281 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/country.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/country.html @@ -1,5 +1,9 @@ {% extends "crisis/crisis-base.html" %} +{# The lines below are for analytics #} +{% block analytics_group_names %}['{{ c.group_dict.name }}']{% endblock %} +{% block analytics_group_ids %}['{{ c.group_dict.id }}']{% endblock %} + {% block subtitle %} {{ c.group_dict.display_name }} {% endblock %} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/custom_country.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/custom_country.html index 39ea417b23..901f0a0e7d 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/custom_country.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/country/custom_country.html @@ -1,5 +1,9 @@ {% extends "crisis/crisis-base.html" %} +{# The lines below are for analytics #} +{% block analytics_group_names %}['{{ data.country_name }}']{% endblock %} +{% block analytics_group_ids %}['{{ data.country_id }}']{% endblock %} + {% block subtitle %}{{ data.country_title }}{% endblock %} {% block crisis_title %} {{ data.country_title }} {% endblock %} {% block crisis_actions %} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/header.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/header.html index b0258892d1..838a172ec3 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/header.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/header.html @@ -141,6 +141,7 @@

+ {% if c.full_facet_info %} {% set filter_icon_class = 'icon-close_filter_button' if c.full_facet_info.get("filters_selected") or c.full_facet_info.get("query_selected") else 'icon-open_filter_button' %} @@ -216,6 +217,7 @@

+

diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/indicator/read.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/indicator/read.html index ea0831d973..3bb88eb083 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/indicator/read.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/indicator/read.html @@ -5,9 +5,14 @@ {% block subtitle %}{{ pkg.title or pkg.name }}{% endblock %} {# The line below is for google analytics #} -{% block ga_dimension1 %}{{ pkg.organization.title }}{% endblock %} -{% block ga_dimension2 %}{{ c.ga_dataset_type }}{% endblock %} -{% block ga_dimension3 %}{{ c.ga_location }}{% endblock %} +{% block analytics_org_name %}{{ pkg.organization.name }}{% endblock %} +{% block analytics_org_id %}{{ pkg.organization.id }}{% endblock %} +{% block analytics_is_cod %}{{ c.analytics_is_cod }}{% endblock %} +{% block analytics_is_indicator %}{{ c.analytics_is_indicator }}{% endblock %} +{% block analytics_group_names %}{{ c.analytics_group_names | safe }}{% endblock %} +{% block analytics_group_ids %}{{ c.analytics_group_ids | safe }}{% endblock %} +{% block analytics_dataset_name %}{{ pkg.name }}{% endblock %} +{% block analytics_dataset_id %}{{ pkg.id }}{% endblock %} {% block breadcrumb_content %} {% block breadcrum_parent_item %} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/custom/custom_org.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/custom/custom_org.html index c0b980a5fb..0aeec0fd88 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/custom/custom_org.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/custom/custom_org.html @@ -1,5 +1,9 @@ {% extends "page.html" %} +{# The lines below are for analytics #} +{% block analytics_org_name %}{{ data.org_info.name }}{% endblock %} +{% block analytics_org_id %}{{ data.org_info.id }}{% endblock %} + {% block subtitle %}{{ data.org_info.display_name }}{% endblock %} {% block breadcrumb_content %} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/edit.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/edit.html index 741c239956..fff86b95e6 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/edit.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/edit.html @@ -1,5 +1,10 @@ {% extends "organization/base_form_page.html" %} +{# The lines below are for analytics #} +{% block analytics_org_name %}{{ data.name }}{% endblock %} +{% block analytics_org_id %}{{ data.id }}{% endblock %} + + {% block subtitle %}{{ _('Edit Organisation') }}{% endblock %} {% block page_heading_class %}hide-heading{% endblock %} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/members.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/members.html index 94b33b718f..909cf6d3cc 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/members.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/members.html @@ -1,5 +1,10 @@ {% extends "organization/read_base.html" %} +{# The line below is for google analytics #} +{% block analytics_org_name %}{{ c.group_dict.name }}{% endblock %} +{% block analytics_org_id %}{{ c.group_dict.id }}{% endblock %} + + {% set roles=h.hdx_member_roles_list() %} {% set authorized = h.check_access('organization_update', {'id': c.group_dict.id}) %} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/read.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/read.html index dd6c87bbbf..14026bbbeb 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/read.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/organization/read.html @@ -1,5 +1,9 @@ {% extends "organization/read_base.html" %} +{# The lines below are for analytics #} +{% block analytics_org_name %}{{ c.group_dict.name }}{% endblock %} +{% block analytics_org_id %}{{ c.group_dict.id }}{% endblock %} + {% set can_create_dataset = h.check_access('package_create', {'organization_id': c.group_dict.id, 'owner_org': c.group_dict.id}) %} {% set can_edit = h.check_access('organization_update', {'id': c.group_dict.id}) %} {% set can_actions = can_create_dataset or can_edit %} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/package/resource_read.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/package/resource_read.html index 9dd2214433..f2fcd036df 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/package/resource_read.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/package/resource_read.html @@ -1,12 +1,16 @@ {% extends "package/base.html" %} {% set res = c.resource %} -{% set resource_dwd_url = res.perma_link if res.perma_link else res.url %} -{# The line below is for google analytics #} -{% block ga_dimension1 %}{{ c.package.organization.title }}{% endblock %} -{% block ga_dimension2 %}{{ c.ga_dataset_type }}{% endblock %} -{% block ga_dimension3 %}{{ c.ga_location }}{% endblock %} +{# The lines below are for analytics #} +{% block analytics_org_name %}{{ c.package.organization.name }}{% endblock %} +{% block analytics_org_id %}{{ c.package.organization.id }}{% endblock %} +{% block analytics_is_cod %}{{ c.analytics_is_cod }}{% endblock %} +{% block analytics_is_indicator %}{{ c.analytics_is_indicator }}{% endblock %} +{% block analytics_group_names %}{{ c.analytics_group_names | safe }}{% endblock %} +{% block analytics_group_ids %}{{ c.analytics_group_ids | safe }}{% endblock %} +{% block analytics_dataset_name %}{{ c.package.name }}{% endblock %} +{% block analytics_dataset_id %}{{ c.package.id }}{% endblock %} {% block head_extras -%} {{ super() }} @@ -44,7 +48,7 @@ #} {% if res.url and h.is_url(res.url) %}
  • - + {# {% if res.resource_type in ('listing', 'service') %} {{ _('View') }} @@ -58,7 +62,7 @@ #} {{ _('Download') }} - +
    • {% endif %} @@ -73,9 +77,9 @@ {% block resource_read_title %}

    {{ h.resource_display_name(res) | truncate(50) }}

    {% endblock %} {% block resource_read_url %} {% if res.url and h.is_url(res.url) %} -

    {{ _('URL:') }} {{ resource_dwd_url }}

    +

    {{ _('URL:') }} {{ res.original_url }}

    {% elif res.url %} -

    {{ _('URL:') }} {{ resource_dwd_url }}

    +

    {{ _('URL:') }} {{ res.download_url }}

    {% endif %} {% endblock %}
    diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/package/snippets/resource_item.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/package/snippets/resource_item.html index 6fea3a7593..439fc7f21f 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/package/snippets/resource_item.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/package/snippets/resource_item.html @@ -73,7 +73,7 @@ {{ _('Download') }} - + {% set button_id = 'social-btn-' + res.id %} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/search/snippets/package_list.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/search/snippets/package_list.html index c65c8a0e34..1990b74848 100644 --- a/ckanext-hdx_theme/ckanext/hdx_theme/templates/search/snippets/package_list.html +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/search/snippets/package_list.html @@ -20,6 +20,9 @@ {% set min_version = full_facet_info.get("filters_selected") or full_facet_info.get("query_selected") %} {% set searchValue = request.params.get('q') if request.params.get('q') else '' %} + + + {% if packages %} @@ -154,6 +157,9 @@ {% else %}
    {% set src_msg = 'for \"'+ searchValue +'\"' if searchValue else '' %} + + +

    Sorry no datasets found {{src_msg}}

    {% endif %} diff --git a/ckanext-hdx_theme/ckanext/hdx_theme/templates/snippets/mixpanel.html b/ckanext-hdx_theme/ckanext/hdx_theme/templates/snippets/mixpanel.html new file mode 100644 index 0000000000..0db3f42007 --- /dev/null +++ b/ckanext-hdx_theme/ckanext/hdx_theme/templates/snippets/mixpanel.html @@ -0,0 +1,44 @@ + + + \ No newline at end of file diff --git a/ckanext-hdx_theme/docs/README.rst b/ckanext-hdx_theme/docs/README.rst new file mode 100644 index 0000000000..aa2f63bbcd --- /dev/null +++ b/ckanext-hdx_theme/docs/README.rst @@ -0,0 +1,6 @@ +HDX DOCUMENTATION +================= + +ANALYTICS +--------- +Information about how we collect analytics information can be found here `Analytics Documentation `_ \ No newline at end of file diff --git a/ckanext-hdx_theme/docs/analytics/index.rst b/ckanext-hdx_theme/docs/analytics/index.rst new file mode 100644 index 0000000000..67b00a3302 --- /dev/null +++ b/ckanext-hdx_theme/docs/analytics/index.rst @@ -0,0 +1,20 @@ +ANALYTICS IN HDX +================ + +Useful information about HDX and Mixpanel +----------------------------------------- +There will be 3 mixpanel projects: + +#. For the production server +#. For the testing/feature servers +#. For local servers running on dev's computers + + +Setup steps +----------- + +#. The **mixpanel token** needs to be setup in prod.ini under :code:`hdx.analytics.mixpanel.token` +#. The **gislayer** needs to be setup with the latest code. It's dependencies were updated as well ( *requirements.txt* ) + + * | there is a new queue for processing analytics requests called **analytics_q**. So we need a rq worker listening on it. + We can either use a separate worker for each queue or workers that listen on both queues. diff --git a/common-config-ini.txt b/common-config-ini.txt index 2bdbbe1095..8fb6e29d33 100644 --- a/common-config-ini.txt +++ b/common-config-ini.txt @@ -204,6 +204,12 @@ hdx.google.dev_key = # hdx.gis.layer_import_url = http://localhost:1234/api/add-layer/dataset/{dataset_id}/resource/{resource_id}?resource_download_url={resource_download_url}&url_type={url_type} # hdx.gis.resource_pbf_url = http://localhost:7101/services/postgis/{resource_id}/wkb_geometry/vector-tiles/{z}/{x}/{y}.pbf +# Analytics +hdx.analytics.ga.token = UA-48221887-3 +# This should be overriden in your own prod.ini +# hdx.analytics.enqueue_url = http://localhost:1234/api/send-analytics +hdx.analytics.mixpanel.token = 875bfe50f9cb981f4e2817832c83c165 + hdx.captcha.url = https://www.google.com/recaptcha/api/siteverify hdx.onboarding.send_confirmation_email = true