research/news-signals-example-notebooks/Search_Aylien_NewsAPI_Using_Entity_Name.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "69084fb1",
   "metadata": {},
   "source": [
    "### GOALS:     \n",
    "- find a wikidata entity by name\n",
    "- look at the volume timeseries for the entity\n",
    "- understand the stories that caused any anomalies\n",
    "- save the signal for future investigation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34fcbf5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# not needed if news_signals is already installed\n",
    "# you might see a pip version error but it's grand, don't worry\n",
    "!pip install -q news_signals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "589468a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "\n",
    "from news_signals import signals, newsapi, wikidata_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30a3e7cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a trial account here: https://aylien.com/news-api-signup\n",
    "# then go to https://app.aylien.com/dashboard to get your credentials\n",
    "\n",
    "# note it's better to set these as environment variables if you know how, and delete this cell\n",
    "# so that API keys aren't accidentally passed around\n",
    "NEWSAPI_APP_ID='<set-this>'\n",
    "NEWSAPI_APP_KEY='<set-this>'\n",
    "newsapi.set_headers(NEWSAPI_APP_ID, NEWSAPI_APP_KEY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "532ea9ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's setup the entity we want to work with\n",
    "entity_name = 'OpenAI'\n",
    "\n",
    "entity_id_candidates = wikidata_utils.search_wikidata(entity_name)\n",
    "test_entity = entity_id_candidates[0]\n",
    "test_entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c19643f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# cool, now let's create a signal\n",
    "signal = signals.AylienSignal(\n",
    "    name=test_entity['label'],\n",
    "    params={\"entity_ids\": [test_entity['id']]}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffe5a7c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's instantiate our signal for the time period we care about\n",
    "start = '2023-01-01'\n",
    "end = '2023-02-01'\n",
    "\n",
    "\n",
    "timeseries_signal = signal(start, end)\n",
    "timeseries_signal.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9f727c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# did the signal have any unexpected spikes?\n",
    "anomaly_signal = signal.anomaly_signal()\n",
    "anomaly_signal.anomalies.plot(color='red')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8a40931c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's have a look at the biggest anomaly\n",
    "highest_anomaly_day = signal.anomalies.idxmax()\n",
    "highest_anomaly_day"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c091ceee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# what was going on that day?\n",
    "signal = signal.sample_stories_in_window(\n",
    "    start=highest_anomaly_day,\n",
    "    end=highest_anomaly_day + datetime.timedelta(days=1)\n",
    ")\n",
    "\n",
    "for s in signal.feeds_df.stories[0]:\n",
    "    print(s['title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c57d9c10",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "\n",
    "signal_storage_dir = Path('tmp/signal_store')\n",
    "signal_storage_dir.mkdir(exist_ok=True, parents=True)\n",
    "\n",
    "signal.save(signal_storage_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "614cafb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "reloaded_signal = signals.Signal.load(signal_storage_dir)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4ecb0fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "reloaded_signal.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89bc3003",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "print('Title Entities:')\n",
    "for stories in reloaded_signal['stories']:\n",
    "    if type(stories) is np.ndarray:\n",
    "        for s in stories:\n",
    "            for e in s['entities']:\n",
    "                for m in e['title']['surface_forms']:\n",
    "                    print(m['text'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}