In [1]:
%matplotlib inline
In [2]:
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

Lab11A: Parallel ProcessingΒΆ

1. (30 points) Accelerating network bound procedures.

Python code to show PNG images available on the URL http://people.duke.edu/~ccc14/misc/ is provided.

  • Write a funciton download_one(url, path) that downloads one image given a url and saves the image to path (5 points)

For the exercises below, save the image file to the local directory using the same fileanme as on http://people.duke.edu/~ccc14/misc/. You can just use the %%time magic to time the downloads.

  • Write a for loop to download all images and time how long it takes (5 points)
  • Write a function that uses concurrent.futures and a thread pool with 4 threads to download all images and time how long it takes (5 points)
  • Write a function that uses multiprocessing and a process pool with 4 processes to download all images and time how long it takes (5 points)
  • Write a function that uses ipyparallel and a process pool with 4 processes to download all images and time how long it takes (10 points)
In [3]:
import requests
from bs4 import BeautifulSoup

def listFD(url, ext=''):
    page = requests.get(url).text
    soup = BeautifulSoup(page, 'html.parser')
    return [url + node.get('href') for node in soup.find_all('a')
            if node.get('href').endswith(ext)]

site = 'http://people.duke.edu/~ccc14/misc/'
ext = 'png'
for i, file in enumerate(listFD(site, ext)):
    print(file)
http://people.duke.edu/~ccc14/misc/250px-002Ivysaur.png
http://people.duke.edu/~ccc14/misc/250px-003Venusaur.png
http://people.duke.edu/~ccc14/misc/250px-004Charmander.png
http://people.duke.edu/~ccc14/misc/250px-005Charmeleon.png
http://people.duke.edu/~ccc14/misc/250px-006Charizard.png
http://people.duke.edu/~ccc14/misc/250px-008Wartortle.png
http://people.duke.edu/~ccc14/misc/250px-009Blastoise.png
http://people.duke.edu/~ccc14/misc/250px-012Butterfree.png
http://people.duke.edu/~ccc14/misc/250px-015Beedrill.png
http://people.duke.edu/~ccc14/misc/250px-016Pidgey.png
http://people.duke.edu/~ccc14/misc/250px-017Pidgeotto.png
http://people.duke.edu/~ccc14/misc/250px-018Pidgeot.png
http://people.duke.edu/~ccc14/misc/250px-019Rattata.png
http://people.duke.edu/~ccc14/misc/250px-020Raticate.png
http://people.duke.edu/~ccc14/misc/250px-021Spearow.png
http://people.duke.edu/~ccc14/misc/250px-022Fearow.png
http://people.duke.edu/~ccc14/misc/250px-023Ekans.png
http://people.duke.edu/~ccc14/misc/250px-024Arbok.png
http://people.duke.edu/~ccc14/misc/250px-025Pikachu.png
http://people.duke.edu/~ccc14/misc/250px-026Raichu.png
http://people.duke.edu/~ccc14/misc/250px-028Sandslash.png
http://people.duke.edu/~ccc14/misc/250px-030Nidorina.png
http://people.duke.edu/~ccc14/misc/250px-031Nidoqueen.png
http://people.duke.edu/~ccc14/misc/250px-033Nidorino.png
http://people.duke.edu/~ccc14/misc/250px-034Nidoking.png
http://people.duke.edu/~ccc14/misc/250px-036Clefable.png
http://people.duke.edu/~ccc14/misc/250px-037Vulpix.png
http://people.duke.edu/~ccc14/misc/250px-038Ninetales.png
http://people.duke.edu/~ccc14/misc/250px-041Zubat.png
http://people.duke.edu/~ccc14/misc/250px-042Golbat.png
http://people.duke.edu/~ccc14/misc/250px-049Venomoth.png
http://people.duke.edu/~ccc14/misc/250px-050Diglett.png
http://people.duke.edu/~ccc14/misc/250px-051Dugtrio.png
http://people.duke.edu/~ccc14/misc/250px-052Meowth.png
http://people.duke.edu/~ccc14/misc/250px-053Persian.png
http://people.duke.edu/~ccc14/misc/250px-054Psyduck.png
http://people.duke.edu/~ccc14/misc/250px-055Golduck.png
http://people.duke.edu/~ccc14/misc/250px-056Mankey.png
http://people.duke.edu/~ccc14/misc/250px-057Primeape.png
http://people.duke.edu/~ccc14/misc/250px-058Growlithe.png
http://people.duke.edu/~ccc14/misc/250px-059Arcanine.png
http://people.duke.edu/~ccc14/misc/250px-060Poliwag.png
http://people.duke.edu/~ccc14/misc/250px-061Poliwhirl.png
http://people.duke.edu/~ccc14/misc/250px-062Poliwrath.png
http://people.duke.edu/~ccc14/misc/250px-063Abra.png
http://people.duke.edu/~ccc14/misc/250px-064Kadabra.png
http://people.duke.edu/~ccc14/misc/250px-065Alakazam.png
http://people.duke.edu/~ccc14/misc/250px-068Machamp.png
http://people.duke.edu/~ccc14/misc/250px-070Weepinbell.png
http://people.duke.edu/~ccc14/misc/250px-071Victreebel.png
http://people.duke.edu/~ccc14/misc/250px-072Tentacool.png
http://people.duke.edu/~ccc14/misc/250px-073Tentacruel.png
http://people.duke.edu/~ccc14/misc/250px-077Ponyta.png
http://people.duke.edu/~ccc14/misc/250px-078Rapidash.png
http://people.duke.edu/~ccc14/misc/250px-082Magneton.png
http://people.duke.edu/~ccc14/misc/250px-083Farfetch'd.png
http://people.duke.edu/~ccc14/misc/250px-084Doduo.png
http://people.duke.edu/~ccc14/misc/250px-085Dodrio.png
http://people.duke.edu/~ccc14/misc/250px-087Dewgong.png
http://people.duke.edu/~ccc14/misc/250px-091Cloyster.png
http://people.duke.edu/~ccc14/misc/250px-092Gastly.png
http://people.duke.edu/~ccc14/misc/250px-093Haunter.png
http://people.duke.edu/~ccc14/misc/250px-094Gengar.png
http://people.duke.edu/~ccc14/misc/250px-095Onix.png
http://people.duke.edu/~ccc14/misc/250px-097Hypno.png
http://people.duke.edu/~ccc14/misc/250px-099Kingler.png
http://people.duke.edu/~ccc14/misc/250px-100Voltorb.png
http://people.duke.edu/~ccc14/misc/250px-101Electrode.png
http://people.duke.edu/~ccc14/misc/250px-103Exeggutor.png
http://people.duke.edu/~ccc14/misc/250px-106Hitmonlee.png
http://people.duke.edu/~ccc14/misc/250px-107Hitmonchan.png
http://people.duke.edu/~ccc14/misc/250px-110Weezing.png
http://people.duke.edu/~ccc14/misc/250px-114Tangela.png
http://people.duke.edu/~ccc14/misc/250px-115Kangaskhan.png
http://people.duke.edu/~ccc14/misc/250px-116Horsea.png
http://people.duke.edu/~ccc14/misc/250px-117Seadra.png
http://people.duke.edu/~ccc14/misc/250px-118Goldeen.png
http://people.duke.edu/~ccc14/misc/250px-119Seaking.png
http://people.duke.edu/~ccc14/misc/250px-120Staryu.png
http://people.duke.edu/~ccc14/misc/250px-121Starmie.png
http://people.duke.edu/~ccc14/misc/250px-122Mr._Mime.png
http://people.duke.edu/~ccc14/misc/250px-123Scyther.png
http://people.duke.edu/~ccc14/misc/250px-124Jynx.png
http://people.duke.edu/~ccc14/misc/250px-125Electabuzz.png
http://people.duke.edu/~ccc14/misc/250px-126Magmar.png
http://people.duke.edu/~ccc14/misc/250px-127Pinsir.png
http://people.duke.edu/~ccc14/misc/250px-128Tauros.png
http://people.duke.edu/~ccc14/misc/250px-129Magikarp.png
http://people.duke.edu/~ccc14/misc/250px-130Gyarados.png
http://people.duke.edu/~ccc14/misc/250px-131Lapras.png
http://people.duke.edu/~ccc14/misc/250px-133Eevee.png
http://people.duke.edu/~ccc14/misc/250px-134Vaporeon.png
http://people.duke.edu/~ccc14/misc/250px-135Jolteon.png
http://people.duke.edu/~ccc14/misc/250px-136Flareon.png
http://people.duke.edu/~ccc14/misc/250px-139Omastar.png
http://people.duke.edu/~ccc14/misc/250px-140Kabuto.png
http://people.duke.edu/~ccc14/misc/250px-141Kabutops.png
http://people.duke.edu/~ccc14/misc/250px-142Aerodactyl.png
http://people.duke.edu/~ccc14/misc/250px-144Articuno.png
http://people.duke.edu/~ccc14/misc/250px-145Zapdos.png
http://people.duke.edu/~ccc14/misc/250px-146Moltres.png
http://people.duke.edu/~ccc14/misc/250px-148Dragonair.png
http://people.duke.edu/~ccc14/misc/250px-149Dragonite.png
http://people.duke.edu/~ccc14/misc/250px-150Mewtwo.png
http://people.duke.edu/~ccc14/misc/250px-151Mew.png

Function to download a single image at URL to PATH

In [4]:




Using for loop

In [4]:




Using a thread pool

In [4]:




Using a process pool

In [4]:




Using ``ipyparallel``

In [4]: