From 90e99a56799653334d57ce98eb516aa78ec69985 Mon Sep 17 00:00:00 2001 From: Egor Marin Date: Thu, 27 Jul 2023 17:31:11 +0300 Subject: [PATCH] Delete parallel_analysis_demo.ipynb --- parallel_analysis_demo.ipynb | 443 ----------------------------------- 1 file changed, 443 deletions(-) delete mode 100644 parallel_analysis_demo.ipynb diff --git a/parallel_analysis_demo.ipynb b/parallel_analysis_demo.ipynb deleted file mode 100644 index 84ce08e41da..00000000000 --- a/parallel_analysis_demo.ipynb +++ /dev/null @@ -1,443 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/marinegor/github/mdanalysis/venv_dask/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], - "source": [ - "import MDAnalysis as mda\n", - "from MDAnalysis.analysis import rms\n", - "\n", - "prefix = \"/home/marinegor/modelling/destab/2020-08-06_MD\"\n", - "traj, top = f\"{prefix}/md_0_1.xtc\", f\"{prefix}/md_0_1.gro\"\n", - "\n", - "u = mda.Universe(top, traj)" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "ename": "ValueError", - "evalue": "backend='multiprocessing' is not in self.available_backends=['local'] for class AverageStructure", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[2], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[39mfrom\u001b[39;00m \u001b[39mMDAnalysis\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39manalysis\u001b[39;00m\u001b[39m.\u001b[39;00m\u001b[39malign\u001b[39;00m \u001b[39mimport\u001b[39;00m AverageStructure\n\u001b[0;32m----> 2\u001b[0m avg \u001b[39m=\u001b[39m AverageStructure(mobile\u001b[39m=\u001b[39;49mu)\u001b[39m.\u001b[39;49mrun(backend\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39mmultiprocessing\u001b[39;49m\u001b[39m'\u001b[39;49m, n_workers\u001b[39m=\u001b[39;49m\u001b[39m16\u001b[39;49m)\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:673\u001b[0m, in \u001b[0;36mAnalysisBase.run\u001b[0;34m(self, start, stop, step, frames, verbose, n_workers, n_parts, backend, client, progressbar_kwargs)\u001b[0m\n\u001b[1;32m 670\u001b[0m backend \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m \u001b[39mif\u001b[39;00m client \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39melse\u001b[39;00m backend\n\u001b[1;32m 672\u001b[0m \u001b[39m# do this as early as possible to check client parameters before any computations occur\u001b[39;00m\n\u001b[0;32m--> 673\u001b[0m client \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_configure_client(backend\u001b[39m=\u001b[39;49mbackend, n_workers\u001b[39m=\u001b[39;49mn_workers, client\u001b[39m=\u001b[39;49mclient)\n\u001b[1;32m 675\u001b[0m \u001b[39m# start preparing the run\u001b[39;00m\n\u001b[1;32m 676\u001b[0m worker_func \u001b[39m=\u001b[39m partial(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compute, progressbar_kwargs\u001b[39m=\u001b[39mprogressbar_kwargs, verbose\u001b[39m=\u001b[39mverbose)\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:599\u001b[0m, in \u001b[0;36mAnalysisBase._configure_client\u001b[0;34m(self, backend, n_workers, client)\u001b[0m\n\u001b[1;32m 585\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 586\u001b[0m \u001b[39mConfigure parameters necessary for running a distributed workload.\u001b[39;00m\n\u001b[1;32m 587\u001b[0m \n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 596\u001b[0m \u001b[39m number of workers (local or remote processes).\u001b[39;00m\n\u001b[1;32m 597\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 598\u001b[0m \u001b[39mif\u001b[39;00m backend \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m backend \u001b[39mnot\u001b[39;00m \u001b[39min\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mavailable_backends:\n\u001b[0;32m--> 599\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mbackend\u001b[39m=}\u001b[39;00m\u001b[39m is not in \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mavailable_backends\u001b[39m=}\u001b[39;00m\u001b[39m for class \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__class__\u001b[39m\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 600\u001b[0m \u001b[39mreturn\u001b[39;00m Client(backend\u001b[39m=\u001b[39mbackend, n_workers\u001b[39m=\u001b[39mn_workers, client\u001b[39m=\u001b[39mclient)\n", - "\u001b[0;31mValueError\u001b[0m: backend='multiprocessing' is not in self.available_backends=['local'] for class AverageStructure" - ] - } - ], - "source": [ - "from MDAnalysis.analysis.align import AverageStructure\n", - "avg = AverageStructure(mobile=u).run(backend='multiprocessing', n_workers=16)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['local']" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "AverageStructure.available_backends" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 4.77 s, sys: 15.5 ms, total: 4.79 s\n", - "Wall time: 4.79 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "from MDAnalysis.analysis.align import AverageStructure\n", - "avg = AverageStructure(mobile=u).run(step=100)\n", - "\n", - "ref = avg.results.universe" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "groupselections = (\"protein\", \"backbone\", \"name CA\")\n", - "\n", - "R = rms.RMSD(\n", - " u, # universe to align\n", - " ref, # reference universe or atomgroup\n", - " groupselections=groupselections,\n", - " # select=\"backbone\", # group to superimpose and calculate RMSD\n", - ")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m:1\u001b[0m\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:679\u001b[0m, in \u001b[0;36mAnalysisBase.run\u001b[0;34m(self, start, stop, step, frames, verbose, n_workers, n_parts, backend, client, progressbar_kwargs)\u001b[0m\n\u001b[1;32m 677\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_prepare()\n\u001b[1;32m 678\u001b[0m computation_groups \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_setup_computation_groups(start\u001b[39m=\u001b[39mstart, stop\u001b[39m=\u001b[39mstop, step\u001b[39m=\u001b[39mstep, frames\u001b[39m=\u001b[39mframes, n_parts\u001b[39m=\u001b[39mn_parts)\n\u001b[0;32m--> 679\u001b[0m remote_results: \u001b[39mlist\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39mAnalysisBase\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39;49mapply(worker_func, computation_groups)\n\u001b[1;32m 680\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mresults \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_parallel_conclude(remote_results\u001b[39m=\u001b[39mremote_results)\n\u001b[1;32m 682\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_conclude()\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:188\u001b[0m, in \u001b[0;36mClient.apply\u001b[0;34m(self, func, computations)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mapply\u001b[39m(\u001b[39mself\u001b[39m, func: Callable, computations: \u001b[39mlist\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mlist\u001b[39m:\n\u001b[1;32m 187\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbackend \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mlocal\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[0;32m--> 188\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_compute_with_local(func, computations)\n\u001b[1;32m 189\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbackend \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mmultiprocessing\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m 190\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compute_with_multiprocessing(func, computations)\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:199\u001b[0m, in \u001b[0;36mClient._compute_with_local\u001b[0;34m(self, func, computations)\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_compute_with_local\u001b[39m(\u001b[39mself\u001b[39m, func: Callable, computations: \u001b[39mlist\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mlist\u001b[39m:\n\u001b[0;32m--> 199\u001b[0m \u001b[39mreturn\u001b[39;00m [func(task) \u001b[39mfor\u001b[39;00m task \u001b[39min\u001b[39;00m computations]\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:199\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_compute_with_local\u001b[39m(\u001b[39mself\u001b[39m, func: Callable, computations: \u001b[39mlist\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mlist\u001b[39m:\n\u001b[0;32m--> 199\u001b[0m \u001b[39mreturn\u001b[39;00m [func(task) \u001b[39mfor\u001b[39;00m task \u001b[39min\u001b[39;00m computations]\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:531\u001b[0m, in \u001b[0;36mAnalysisBase._compute\u001b[0;34m(self, indexed_frames, verbose, progressbar_kwargs)\u001b[0m\n\u001b[1;32m 529\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mframes[i] \u001b[39m=\u001b[39m ts\u001b[39m.\u001b[39mframe\n\u001b[1;32m 530\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtimes[i] \u001b[39m=\u001b[39m ts\u001b[39m.\u001b[39mtime\n\u001b[0;32m--> 531\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_single_frame()\n\u001b[1;32m 532\u001b[0m logger\u001b[39m.\u001b[39minfo(\u001b[39m\"\u001b[39m\u001b[39mFinishing up\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 533\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/rms.py:699\u001b[0m, in \u001b[0;36mRMSD._single_frame\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 696\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ts\u001b[39m.\u001b[39mpositions[:] \u001b[39m-\u001b[39m\u001b[39m=\u001b[39m mobile_com\n\u001b[1;32m 698\u001b[0m \u001b[39m# R acts to the left & is broadcasted N times.\u001b[39;00m\n\u001b[0;32m--> 699\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ts\u001b[39m.\u001b[39mpositions[:] \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39;49mdot(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_ts\u001b[39m.\u001b[39;49mpositions, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_R)\n\u001b[1;32m 700\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ts\u001b[39m.\u001b[39mpositions \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ref_com\n\u001b[1;32m 702\u001b[0m \u001b[39m# 2) calculate secondary RMSDs (without any further\u001b[39;00m\n\u001b[1;32m 703\u001b[0m \u001b[39m# superposition)\u001b[39;00m\n", - "File \u001b[0;32m<__array_function__ internals>:200\u001b[0m, in \u001b[0;36mdot\u001b[0;34m(*args, **kwargs)\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "%%time\n", - "\n", - "R.run()" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 8%|▊ | 7713/100001 [00:21<04:16, 360.32it/s]\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m R\u001b[39m.\u001b[39;49mrun(verbose\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:679\u001b[0m, in \u001b[0;36mAnalysisBase.run\u001b[0;34m(self, start, stop, step, frames, verbose, n_workers, n_parts, backend, client, progressbar_kwargs)\u001b[0m\n\u001b[1;32m 677\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_prepare()\n\u001b[1;32m 678\u001b[0m computation_groups \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_setup_computation_groups(start\u001b[39m=\u001b[39mstart, stop\u001b[39m=\u001b[39mstop, step\u001b[39m=\u001b[39mstep, frames\u001b[39m=\u001b[39mframes, n_parts\u001b[39m=\u001b[39mn_parts)\n\u001b[0;32m--> 679\u001b[0m remote_results: \u001b[39mlist\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39mAnalysisBase\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39;49mapply(worker_func, computation_groups)\n\u001b[1;32m 680\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mresults \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_parallel_conclude(remote_results\u001b[39m=\u001b[39mremote_results)\n\u001b[1;32m 682\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_conclude()\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:188\u001b[0m, in \u001b[0;36mClient.apply\u001b[0;34m(self, func, computations)\u001b[0m\n\u001b[1;32m 186\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mapply\u001b[39m(\u001b[39mself\u001b[39m, func: Callable, computations: \u001b[39mlist\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mlist\u001b[39m:\n\u001b[1;32m 187\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbackend \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mlocal\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[0;32m--> 188\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_compute_with_local(func, computations)\n\u001b[1;32m 189\u001b[0m \u001b[39melif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbackend \u001b[39m==\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mmultiprocessing\u001b[39m\u001b[39m\"\u001b[39m:\n\u001b[1;32m 190\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compute_with_multiprocessing(func, computations)\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:199\u001b[0m, in \u001b[0;36mClient._compute_with_local\u001b[0;34m(self, func, computations)\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_compute_with_local\u001b[39m(\u001b[39mself\u001b[39m, func: Callable, computations: \u001b[39mlist\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mlist\u001b[39m:\n\u001b[0;32m--> 199\u001b[0m \u001b[39mreturn\u001b[39;00m [func(task) \u001b[39mfor\u001b[39;00m task \u001b[39min\u001b[39;00m computations]\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:199\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_compute_with_local\u001b[39m(\u001b[39mself\u001b[39m, func: Callable, computations: \u001b[39mlist\u001b[39m) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mlist\u001b[39m:\n\u001b[0;32m--> 199\u001b[0m \u001b[39mreturn\u001b[39;00m [func(task) \u001b[39mfor\u001b[39;00m task \u001b[39min\u001b[39;00m computations]\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:531\u001b[0m, in \u001b[0;36mAnalysisBase._compute\u001b[0;34m(self, indexed_frames, verbose, progressbar_kwargs)\u001b[0m\n\u001b[1;32m 529\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mframes[i] \u001b[39m=\u001b[39m ts\u001b[39m.\u001b[39mframe\n\u001b[1;32m 530\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtimes[i] \u001b[39m=\u001b[39m ts\u001b[39m.\u001b[39mtime\n\u001b[0;32m--> 531\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_single_frame()\n\u001b[1;32m 532\u001b[0m logger\u001b[39m.\u001b[39minfo(\u001b[39m\"\u001b[39m\u001b[39mFinishing up\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m 533\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/rms.py:705\u001b[0m, in \u001b[0;36mRMSD._single_frame\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ts\u001b[39m.\u001b[39mpositions \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_ref_com\n\u001b[1;32m 702\u001b[0m \u001b[39m# 2) calculate secondary RMSDs (without any further\u001b[39;00m\n\u001b[1;32m 703\u001b[0m \u001b[39m# superposition)\u001b[39;00m\n\u001b[1;32m 704\u001b[0m \u001b[39mfor\u001b[39;00m igroup, (refpos, atoms) \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(\n\u001b[0;32m--> 705\u001b[0m \u001b[39mzip\u001b[39;49m(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_groupselections_ref_coords64,\n\u001b[1;32m 706\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_groupselections_atoms), \u001b[39m3\u001b[39m):\n\u001b[1;32m 707\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mresults\u001b[39m.\u001b[39mrmsd[\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_frame_index, igroup] \u001b[39m=\u001b[39m rmsd(\n\u001b[1;32m 708\u001b[0m refpos, atoms[\u001b[39m'\u001b[39m\u001b[39mmobile\u001b[39m\u001b[39m'\u001b[39m]\u001b[39m.\u001b[39mpositions,\n\u001b[1;32m 709\u001b[0m weights\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mweights_groupselections[igroup\u001b[39m-\u001b[39m\u001b[39m3\u001b[39m],\n\u001b[1;32m 710\u001b[0m center\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m, superposition\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m)\n\u001b[1;32m 711\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 712\u001b[0m \u001b[39m# only calculate RMSD by setting the Rmatrix to None (no need\u001b[39;00m\n\u001b[1;32m 713\u001b[0m \u001b[39m# to carry out the rotation as we already get the optimum RMSD)\u001b[39;00m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " - ] - } - ], - "source": [ - "R.run(verbose=True)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "('local', 'dask', 'multiprocessing', 'dask.distributed')" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "R.available_backends" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "self.n_workers=4, len(computations)=4\n", - "CPU times: user 101 ms, sys: 55.9 ms, total: 156 ms\n", - "Wall time: 1min 14s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "R_multiprocessing = R.run(\n", - " backend='multiprocessing', n_workers=4\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "self.n_workers=16, len(computations)=16\n", - "CPU times: user 630 ms, sys: 258 ms, total: 888 ms\n", - "Wall time: 54.4 s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "R_multiprocessing = R.run(\n", - " backend='multiprocessing', n_workers=16\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "self.n_workers=16, len(computations)=16\n", - "CPU times: user 493 ms, sys: 314 ms, total: 807 ms\n", - "Wall time: 1min 48s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "R_dask = R.run(\n", - " backend='dask', n_workers=16\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "ename": "ValueError", - "evalue": "Backend or client is not set properly: self.backend='dask.distributed', self.client=None", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m:1\u001b[0m\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:679\u001b[0m, in \u001b[0;36mAnalysisBase.run\u001b[0;34m(self, start, stop, step, frames, verbose, n_workers, n_parts, backend, client, progressbar_kwargs)\u001b[0m\n\u001b[1;32m 677\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_prepare()\n\u001b[1;32m 678\u001b[0m computation_groups \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_setup_computation_groups(start\u001b[39m=\u001b[39mstart, stop\u001b[39m=\u001b[39mstop, step\u001b[39m=\u001b[39mstep, frames\u001b[39m=\u001b[39mframes, n_parts\u001b[39m=\u001b[39mn_parts)\n\u001b[0;32m--> 679\u001b[0m remote_results: \u001b[39mlist\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39mAnalysisBase\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m client\u001b[39m.\u001b[39;49mapply(worker_func, computation_groups)\n\u001b[1;32m 680\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mresults \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_parallel_conclude(remote_results\u001b[39m=\u001b[39mremote_results)\n\u001b[1;32m 682\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_conclude()\n", - "File \u001b[0;32m~/github/mdanalysis/package/MDAnalysis/analysis/base.py:196\u001b[0m, in \u001b[0;36mClient.apply\u001b[0;34m(self, func, computations)\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_compute_with_client(func, computations)\n\u001b[1;32m 195\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 196\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mValueError\u001b[39;00m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mBackend or client is not set properly: \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbackend\u001b[39m=}\u001b[39;00m\u001b[39m, \u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mclient\u001b[39m=}\u001b[39;00m\u001b[39m\"\u001b[39m)\n", - "\u001b[0;31mValueError\u001b[0m: Backend or client is not set properly: self.backend='dask.distributed', self.client=None" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "R_dask = R.run(\n", - " backend='dask.distributed', n_workers=16\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "from dask.distributed import Client, LocalCluster\n", - "\n", - "cluster = LocalCluster(n_workers=16, \n", - " threads_per_worker=1,\n", - " memory_limit='30Gb')\n", - "client = Client(cluster)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/marinegor/github/mdanalysis/venv_dask/lib/python3.10/site-packages/distributed/node.py:182: UserWarning: Port 8787 is already in use.\n", - "Perhaps you already have a cluster running?\n", - "Hosting the HTTP server on port 33445 instead\n", - " warnings.warn(\n" - ] - } - ], - "source": [ - "from dask.distributed import Client\n", - "\n", - "client = Client()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'http://127.0.0.1:8787/status'" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cluster.dashboard_link" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "self.n_workers=16, len(computations)=16\n", - "CPU times: user 24.4 s, sys: 3.5 s, total: 27.9 s\n", - "Wall time: 1min\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "R_distributed = R.run(client=client)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "self.n_workers=16, len(computations)=96\n", - "CPU times: user 25.8 s, sys: 5.56 s, total: 31.4 s\n", - "Wall time: 1min 21s\n" - ] - } - ], - "source": [ - "%%time\n", - "\n", - "R_distributed = R.run(client=client, n_parts=96)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "\n", - "def dropna(arr: np.ndarray) -> np.ndarray:\n", - " return arr[~np.isnan(arr)]\n", - "\n", - "\n", - "runs = {\n", - " \"local\": R,\n", - " \"multiprocessing\": R_multiprocessing,\n", - " \"dask\": R_dask,\n", - " \"dask.distributed\": R_distributed,\n", - "}\n", - "ref_run = R\n", - "\n", - "true_rmsd = dropna(R.results.rmsd)\n", - "for key, run in runs.items():\n", - " rmsd = dropna(run.results.rmsd)\n", - " assert np.all(true_rmsd == rmsd)\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.10.0 ('venv_dask': venv)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.0" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "ad8af1dd7238fd0bdf4d5de60b82692a603f2cfea7f5ed26e9bcefac6229e968" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}