{
 "metadata": {
  "name": "simple_time_consuming_pp"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Basic imports"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os\n",
      "from IPython.core.display import display\n",
      "from IPython import parallel\n",
      "rc = parallel.Client()\n",
      "dv = rc[:]\n",
      "rc.ids"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 2,
       "text": [
        "[0, 1, 2, 3]"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "Sequential Code"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import time\n",
      "t = time.time()\n",
      "\n",
      "def calculate(t):\n",
      "\n",
      "    \"A supposedly time-consuming calculation on 't'.\"\n",
      "\n",
      "    i, j = t\n",
      "    time.sleep(delay)\n",
      "    return i * N + j\n",
      "# Initialise an array.\n",
      "N=5\n",
      "delay=1\n",
      "sequence = []\n",
      "for i in range(0, N):\n",
      "    for j in range(0, N):\n",
      "        sequence.append((i, j))\n",
      "# Perform the work.\n",
      "results = map(calculate, sequence)\n",
      "print results\n",
      "# Show the results.\n",
      "for i in range(0, N):\n",
      "    for result in results[i*N:i*N+N]:\n",
      "            print result,\n",
      "    print\n",
      "print \"Time taken:\", time.time() - t"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]\n",
        "0 1 2 3 4\n",
        "5 6 7 8 9\n",
        "10 11 12 13 14\n",
        "15 16 17 18 19\n",
        "20 21 22 23 24\n",
        "Time taken: 25.006000042\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Parallel Code\n",
      "\n",
      "map_async(self, f, *sequences, **kwargs)\n",
      " |      Parallel version of builtin `map`, using this view's engines.\n",
      " This is equivalent to map(...block=False)\n",
      " |      \n",
      " |      \n",
      " |  \n",
      "map_sync(self, f, *sequences, **kwargs)\n",
      " |      Parallel version of builtin `map`, using this view's engines.\n",
      " This is equivalent to map(...block=True)\n",
      " |      \n",
      " |      "
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%px import time\n",
      "from IPython.core.display import display\n",
      "t = time.time()\n",
      "view=rc.load_balanced_view()\n",
      "view.block=False\n",
      "\n",
      "def calculatep(t):\n",
      "    \"A supposedly time-consuming calculation on 't'.\"\n",
      "\n",
      "    i, j = t\n",
      "    time.sleep(delay)\n",
      "    return i * N + j\n",
      "# Initialise an array.\n",
      "%px N=5\n",
      "%px delay=1\n",
      "sequence = []\n",
      "for i in range(0, N):\n",
      "    for j in range(0, N):\n",
      "        sequence.append((i, j))\n",
      "# Perform the work.\n",
      "results = view.map_async(calculatep, sequence)\n",
      "print list(results)\n",
      "#Show the results.\n",
      "for i in range(0, N):\n",
      "    for result in results[i*N:i*N+N]:\n",
      "           print result,    \n",
      "    print\n",
      "print \"Time taken:\", time.time() - t"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]\n",
        "0 1 2 3 4\n",
        "5 6 7 8 9\n",
        "10 11 12 13 14\n",
        "15 16 17 18 19\n",
        "20 21 22 23 24\n",
        "Time taken: 7.13100004196\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%px print N, delay"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[stdout:0] 5 1\n",
        "[stdout:1] 5 1\n",
        "[stdout:2] 5 1\n",
        "[stdout:3] 5 1\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dv"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "<DirectView [0, 1, 2, 3]>"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rc.ids"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 6,
       "text": [
        "[0, 1, 2, 3]"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "map(display,rc.ids)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "display_data",
       "text": [
        "0"
       ]
      },
      {
       "output_type": "display_data",
       "text": [
        "1"
       ]
      },
      {
       "output_type": "display_data",
       "text": [
        "2"
       ]
      },
      {
       "output_type": "display_data",
       "text": [
        "3"
       ]
      },
      {
       "output_type": "pyout",
       "prompt_number": 7,
       "text": [
        "[None, None, None, None]"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "help(map)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Help on built-in function map in module __builtin__:\n",
        "\n",
        "map(...)\n",
        "    map(function, sequence[, sequence, ...]) -> list\n",
        "    \n",
        "    Return a list of the results of applying the function to the items of\n",
        "    the argument sequence(s).  If more than one sequence is given, the\n",
        "    function is called with an argument list consisting of the corresponding\n",
        "    item of each sequence, substituting None for missing values when not all\n",
        "    sequences have the same length.  If the function is None, return a list of\n",
        "    the items of the sequence (or a list of tuples if more than one sequence).\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "map(function, sequence) calls function(item) for each of the sequence\u2019s items and returns a list of the return values. For example, to compute some cubes:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def cube(x): return x*x*x\n",
      "map(cube, range(1, 11))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "[1, 8, 27, 64, 125, 216, 343, 512, 729, 1000]"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "More than one sequence may be passed; the function must then have as many arguments as there are sequences and is called with the corresponding item from each sequence (or None if some sequence is shorter than another). For example:"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "More than one sequence may be passed; the function must then have as many arguments as there are sequences and is called with the corresponding item from each sequence (or None if some sequence is shorter than another). For example:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "seq = range(8) \n",
      "def add(x, y): return x+y\n",
      "map(add, seq, seq) "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "[0, 2, 4, 6, 8, 10, 12, 14]"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "help(view)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Help on LoadBalancedView in module IPython.parallel.client.view object:\n",
        "\n",
        "class LoadBalancedView(View)\n",
        " |  An load-balancing View that only executes via the Task scheduler.\n",
        " |  \n",
        " |  Load-balanced views can be created with the client's `view` method:\n",
        " |  \n",
        " |  >>> v = client.load_balanced_view()\n",
        " |  \n",
        " |  or targets can be specified, to restrict the potential destinations:\n",
        " |  \n",
        " |  >>> v = client.client.load_balanced_view([1,3])\n",
        " |  \n",
        " |  which would restrict loadbalancing to between engines 1 and 3.\n",
        " |  \n",
        " |  Method resolution order:\n",
        " |      LoadBalancedView\n",
        " |      View\n",
        " |      IPython.utils.traitlets.HasTraits\n",
        " |      __builtin__.object\n",
        " |  \n",
        " |  Methods defined here:\n",
        " |  \n",
        " |  __init__(self, client=None, socket=None, **flags)\n",
        " |  \n",
        " |  map(self, f, *sequences, **kwargs)\n",
        " |      view.map(f, *sequences, block=self.block, chunksize=1, ordered=True) => list|AsyncMapResult\n",
        " |      \n",
        " |      Parallel version of builtin `map`, load-balanced by this View.\n",
        " |      \n",
        " |      `block`, and `chunksize` can be specified by keyword only.\n",
        " |      \n",
        " |      Each `chunksize` elements will be a separate task, and will be\n",
        " |      load-balanced. This lets individual elements be available for iteration\n",
        " |      as soon as they arrive.\n",
        " |      \n",
        " |      Parameters\n",
        " |      ----------\n",
        " |      \n",
        " |      f : callable\n",
        " |          function to be mapped\n",
        " |      *sequences: one or more sequences of matching length\n",
        " |          the sequences to be distributed and passed to `f`\n",
        " |      block : bool [default self.block]\n",
        " |          whether to wait for the result or not\n",
        " |      track : bool\n",
        " |          whether to create a MessageTracker to allow the user to\n",
        " |          safely edit after arrays and buffers during non-copying\n",
        " |          sends.\n",
        " |      chunksize : int [default 1]\n",
        " |          how many elements should be in each task.\n",
        " |      ordered : bool [default True]\n",
        " |          Whether the results should be gathered as they arrive, or enforce\n",
        " |          the order of submission.\n",
        " |          \n",
        " |          Only applies when iterating through AsyncMapResult as results arrive.\n",
        " |          Has no effect when block=True.\n",
        " |      \n",
        " |      Returns\n",
        " |      -------\n",
        " |      \n",
        " |      if block=False:\n",
        " |          AsyncMapResult\n",
        " |              An object like AsyncResult, but which reassembles the sequence of results\n",
        " |              into a single list. AsyncMapResults can be iterated through before all\n",
        " |              results are complete.\n",
        " |          else:\n",
        " |              the result of map(f,*sequences)\n",
        " |  \n",
        " |  set_flags(self, **kwargs)\n",
        " |      set my attribute flags by keyword.\n",
        " |      \n",
        " |      A View is a wrapper for the Client's apply method, but with attributes\n",
        " |      that specify keyword arguments, those attributes can be set by keyword\n",
        " |      argument with this method.\n",
        " |      \n",
        " |      Parameters\n",
        " |      ----------\n",
        " |      \n",
        " |      block : bool\n",
        " |          whether to wait for results\n",
        " |      track : bool\n",
        " |          whether to create a MessageTracker to allow the user to\n",
        " |          safely edit after arrays and buffers during non-copying\n",
        " |          sends.\n",
        " |      \n",
        " |      after : Dependency or collection of msg_ids\n",
        " |          Only for load-balanced execution (targets=None)\n",
        " |          Specify a list of msg_ids as a time-based dependency.\n",
        " |          This job will only be run *after* the dependencies\n",
        " |          have been met.\n",
        " |      \n",
        " |      follow : Dependency or collection of msg_ids\n",
        " |          Only for load-balanced execution (targets=None)\n",
        " |          Specify a list of msg_ids as a location-based dependency.\n",
        " |          This job will only be run on an engine where this dependency\n",
        " |          is met.\n",
        " |      \n",
        " |      timeout : float/int or None\n",
        " |          Only for load-balanced execution (targets=None)\n",
        " |          Specify an amount of time (in seconds) for the scheduler to\n",
        " |          wait for dependencies to be met before failing with a\n",
        " |          DependencyTimeout.\n",
        " |      \n",
        " |      retries : int\n",
        " |          Number of times a task will be retried on failure.\n",
        " |  \n",
        " |  ----------------------------------------------------------------------\n",
        " |  Data descriptors defined here:\n",
        " |  \n",
        " |  after\n",
        " |  \n",
        " |  follow\n",
        " |  \n",
        " |  retries\n",
        " |      An integer trait.\n",
        " |      \n",
        " |      Longs that are unnecessary (<= sys.maxint) are cast to ints.\n",
        " |  \n",
        " |  timeout\n",
        " |      A casting version of the float trait.\n",
        " |  \n",
        " |  ----------------------------------------------------------------------\n",
        " |  Data and other attributes defined here:\n",
        " |  \n",
        " |  skip_doctest = True\n",
        " |  \n",
        " |  ----------------------------------------------------------------------\n",
        " |  Methods inherited from View:\n",
        " |  \n",
        " |  __len__(self)\n",
        " |  \n",
        " |  __repr__(self)\n",
        " |  \n",
        " |  abort(self, jobs=None, targets=None, block=None)\n",
        " |      Abort jobs on my engines.\n",
        " |      \n",
        " |      Parameters\n",
        " |      ----------\n",
        " |      \n",
        " |      jobs : None, str, list of strs, optional\n",
        " |          if None: abort all jobs.\n",
        " |          else: abort specific msg_id(s).\n",
        " |  \n",
        " |  apply(self, f, *args, **kwargs)\n",
        " |      calls f(*args, **kwargs) on remote engines, returning the result.\n",
        " |      \n",
        " |      This method sets all apply flags via this View's attributes.\n",
        " |      \n",
        " |      if self.block is False:\n",
        " |          returns AsyncResult\n",
        " |      else:\n",
        " |          returns actual result of f(*args, **kwargs)\n",
        " |  \n",
        " |  apply_async(self, f, *args, **kwargs)\n",
        " |      calls f(*args, **kwargs) on remote engines in a nonblocking manner.\n",
        " |      \n",
        " |      returns AsyncResult\n",
        " |  \n",
        " |  apply_sync(self, f, *args, **kwargs)\n",
        " |      calls f(*args, **kwargs) on remote engines in a blocking manner,\n",
        " |       returning the result.\n",
        " |      \n",
        " |      returns: actual result of f(*args, **kwargs)\n",
        " |  \n",
        " |  get_result(self, indices_or_msg_ids=None)\n",
        " |      return one or more results, specified by history index or msg_id.\n",
        " |      \n",
        " |      See client.get_result for details.\n",
        " |  \n",
        " |  imap(self, f, *sequences, **kwargs)\n",
        " |      Parallel version of `itertools.imap`.\n",
        " |      \n",
        " |      See `self.map` for details.\n",
        " |  \n",
        " |  map_async(self, f, *sequences, **kwargs)\n",
        " |      Parallel version of builtin `map`, using this view's engines.\n",
        " |      \n",
        " |      This is equivalent to map(...block=False)\n",
        " |      \n",
        " |      See `self.map` for details.\n",
        " |  \n",
        " |  map_sync(self, f, *sequences, **kwargs)\n",
        " |      Parallel version of builtin `map`, using this view's engines.\n",
        " |      \n",
        " |      This is equivalent to map(...block=True)\n",
        " |      \n",
        " |      See `self.map` for details.\n",
        " |  \n",
        " |  parallel(self, dist='b', block=None, **flags)\n",
        " |      Decorator for making a ParallelFunction\n",
        " |  \n",
        " |  purge_results(self, jobs=[], targets=[])\n",
        " |      Instruct the controller to forget specific results.\n",
        " |  \n",
        " |  queue_status(self, targets=None, verbose=False)\n",
        " |      Fetch the Queue status of my engines\n",
        " |  \n",
        " |  remote(self, block=True, **flags)\n",
        " |      Decorator for making a RemoteFunction\n",
        " |  \n",
        " |  shutdown(self, targets=None, restart=False, hub=False, block=None)\n",
        " |      Terminates one or more engine processes, optionally including the hub.\n",
        " |  \n",
        " |  spin(self)\n",
        " |      spin the client, and sync\n",
        " |  \n",
        " |  temp_flags(*args, **kwds)\n",
        " |      temporarily set flags, for use in `with` statements.\n",
        " |      \n",
        " |      See set_flags for permanent setting of flags\n",
        " |      \n",
        " |      Examples\n",
        " |      --------\n",
        " |      \n",
        " |      >>> view.track=False\n",
        " |      ...\n",
        " |      >>> with view.temp_flags(track=True):\n",
        " |      ...    ar = view.apply(dostuff, my_big_array)\n",
        " |      ...    ar.tracker.wait() # wait for send to finish\n",
        " |      >>> view.track\n",
        " |      False\n",
        " |  \n",
        " |  wait(self, jobs=None, timeout=-1)\n",
        " |      waits on one or more `jobs`, for up to `timeout` seconds.\n",
        " |      \n",
        " |      Parameters\n",
        " |      ----------\n",
        " |      \n",
        " |      jobs : int, str, or list of ints and/or strs, or one or more AsyncResult objects\n",
        " |              ints are indices to self.history\n",
        " |              strs are msg_ids\n",
        " |              default: wait on all outstanding messages\n",
        " |      timeout : float\n",
        " |              a time in seconds, after which to give up.\n",
        " |              default is -1, which means no timeout\n",
        " |      \n",
        " |      Returns\n",
        " |      -------\n",
        " |      \n",
        " |      True : when all msg_ids are done\n",
        " |      False : timeout reached, some msg_ids still outstanding\n",
        " |  \n",
        " |  ----------------------------------------------------------------------\n",
        " |  Data descriptors inherited from View:\n",
        " |  \n",
        " |  block\n",
        " |      A boolean (True, False) trait.\n",
        " |  \n",
        " |  client\n",
        " |      A trait whose value must be an instance of a specified class.\n",
        " |      \n",
        " |      The value can also be an instance of a subclass of the specified class.\n",
        " |  \n",
        " |  history\n",
        " |      An instance of a Python list.\n",
        " |  \n",
        " |  outstanding\n",
        " |      An instance of a Python set.\n",
        " |  \n",
        " |  results\n",
        " |      An instance of a Python dict.\n",
        " |  \n",
        " |  targets\n",
        " |  \n",
        " |  track\n",
        " |      A boolean (True, False) trait.\n",
        " |  \n",
        " |  ----------------------------------------------------------------------\n",
        " |  Methods inherited from IPython.utils.traitlets.HasTraits:\n",
        " |  \n",
        " |  on_trait_change(self, handler, name=None, remove=False)\n",
        " |      Setup a handler to be called when a trait changes.\n",
        " |      \n",
        " |      This is used to setup dynamic notifications of trait changes.\n",
        " |      \n",
        " |      Static handlers can be created by creating methods on a HasTraits\n",
        " |      subclass with the naming convention '_[traitname]_changed'.  Thus,\n",
        " |      to create static handler for the trait 'a', create the method\n",
        " |      _a_changed(self, name, old, new) (fewer arguments can be used, see\n",
        " |      below).\n",
        " |      \n",
        " |      Parameters\n",
        " |      ----------\n",
        " |      handler : callable\n",
        " |          A callable that is called when a trait changes.  Its\n",
        " |          signature can be handler(), handler(name), handler(name, new)\n",
        " |          or handler(name, old, new).\n",
        " |      name : list, str, None\n",
        " |          If None, the handler will apply to all traits.  If a list\n",
        " |          of str, handler will apply to all names in the list.  If a\n",
        " |          str, the handler will apply just to that name.\n",
        " |      remove : bool\n",
        " |          If False (the default), then install the handler.  If True\n",
        " |          then unintall it.\n",
        " |  \n",
        " |  trait_metadata(self, traitname, key)\n",
        " |      Get metadata values for trait by key.\n",
        " |  \n",
        " |  trait_names(self, **metadata)\n",
        " |      Get a list of all the names of this classes traits.\n",
        " |  \n",
        " |  traits(self, **metadata)\n",
        " |      Get a list of all the traits of this class.\n",
        " |      \n",
        " |      The TraitTypes returned don't know anything about the values\n",
        " |      that the various HasTrait's instances are holding.\n",
        " |      \n",
        " |      This follows the same algorithm as traits does and does not allow\n",
        " |      for any simple way of specifying merely that a metadata name\n",
        " |      exists, but has any value.  This is because get_metadata returns\n",
        " |      None if a metadata key doesn't exist.\n",
        " |  \n",
        " |  ----------------------------------------------------------------------\n",
        " |  Class methods inherited from IPython.utils.traitlets.HasTraits:\n",
        " |  \n",
        " |  class_trait_names(cls, **metadata) from IPython.utils.traitlets.MetaHasTraits\n",
        " |      Get a list of all the names of this classes traits.\n",
        " |      \n",
        " |      This method is just like the :meth:`trait_names` method, but is unbound.\n",
        " |  \n",
        " |  class_traits(cls, **metadata) from IPython.utils.traitlets.MetaHasTraits\n",
        " |      Get a list of all the traits of this class.\n",
        " |      \n",
        " |      This method is just like the :meth:`traits` method, but is unbound.\n",
        " |      \n",
        " |      The TraitTypes returned don't know anything about the values\n",
        " |      that the various HasTrait's instances are holding.\n",
        " |      \n",
        " |      This follows the same algorithm as traits does and does not allow\n",
        " |      for any simple way of specifying merely that a metadata name\n",
        " |      exists, but has any value.  This is because get_metadata returns\n",
        " |      None if a metadata key doesn't exist.\n",
        " |  \n",
        " |  ----------------------------------------------------------------------\n",
        " |  Static methods inherited from IPython.utils.traitlets.HasTraits:\n",
        " |  \n",
        " |  __new__(cls, **kw)\n",
        " |  \n",
        " |  ----------------------------------------------------------------------\n",
        " |  Data descriptors inherited from IPython.utils.traitlets.HasTraits:\n",
        " |  \n",
        " |  __dict__\n",
        " |      dictionary for instance variables (if defined)\n",
        " |  \n",
        " |  __weakref__\n",
        " |      list of weak references to the object (if defined)\n",
        " |  \n",
        " |  ----------------------------------------------------------------------\n",
        " |  Data and other attributes inherited from IPython.utils.traitlets.HasTraits:\n",
        " |  \n",
        " |  __metaclass__ = <class 'IPython.utils.traitlets.MetaHasTraits'>\n",
        " |      A metaclass for HasTraits.\n",
        " |      \n",
        " |      This metaclass makes sure that any TraitType class attributes are\n",
        " |      instantiated and sets their name attribute.\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "foo()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}