Streaming and functional programming¶
In [4]:
# standard libraries
import math
import os
import gzip
from glob import glob
import itertools as it
# 3rd party libraries
import toolz as tz
import toolz.curried as c
import numpy as np
Understanding itertools¶
repeat¶
In [5]:
for i, item in enumerate(it.repeat('abc')):
print(item)
if i >= 4:
break
abc
abc
abc
abc
abc
In [6]:
list(it.repeat('abc', 5))
Out[6]:
['abc', 'abc', 'abc', 'abc', 'abc']
Replicating behavior using reular Python¶
In [7]:
def repeat(obj, n=None):
if n is None:
while True:
yield obj
else:
for i in range(n):
yield obj
In [8]:
for i, item in enumerate(repeat('abc')):
print(item)
if i >= 4:
break
abc
abc
abc
abc
abc
In [9]:
list(repeat('abc', 5))
Out[9]:
['abc', 'abc', 'abc', 'abc', 'abc']
cycle¶
In [10]:
list(it.islice(it.cycle('abc'), 10))
Out[10]:
['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c', 'a']
Replicating behavior using regular Python¶
In [11]:
def cycle(obj):
while True:
yield from obj
In [12]:
list(it.islice(cycle('abc'), 10))
Out[12]:
['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c', 'a']
count¶
In [13]:
list(it.islice(it.count(), 5))
Out[13]:
[0, 1, 2, 3, 4]
In [14]:
list(it.islice(it.count(2), 5))
Out[14]:
[2, 3, 4, 5, 6]
In [15]:
list(it.islice(it.count(2,2), 5))
Out[15]:
[2, 4, 6, 8, 10]
Replicating behavior using regular Python¶
In [16]:
def count(start=0, step=1):
while True:
yield start
start += step
In [17]:
list(it.islice(it.count(), 5))
Out[17]:
[0, 1, 2, 3, 4]
In [18]:
list(it.islice(count(2), 5))
Out[18]:
[2, 3, 4, 5, 6]
In [19]:
list(it.islice(count(2,2), 5))
Out[19]:
[2, 4, 6, 8, 10]
chain¶
In [20]:
list(it.islice(it.chain(range(3), 'abc', it.count()), 10))
Out[20]:
[0, 1, 2, 'a', 'b', 'c', 0, 1, 2, 3]
Replicating behavior with regular Python¶
In [21]:
def chain(*args):
for arg in args:
yield from arg
In [22]:
list(it.islice(chain(range(3), 'abc', it.count()), 10))
Out[22]:
[0, 1, 2, 'a', 'b', 'c', 0, 1, 2, 3]
chain.from_iterable¶
In [23]:
list(it.islice(it.chain.from_iterable([range(3), 'abc', it.count()]),10))
Out[23]:
[0, 1, 2, 'a', 'b', 'c', 0, 1, 2, 3]
Replicating behavior with regular Python¶
In [24]:
def chain_from_iterable(args):
for arg in args:
yield from arg
In [25]:
list(it.islice(chain_from_iterable([range(3), 'abc', it.count()]),10))
Out[25]:
[0, 1, 2, 'a', 'b', 'c', 0, 1, 2, 3]
product¶
In [26]:
list(it.product('ab', range(2), 'AB'))
Out[26]:
[('a', 0, 'A'),
('a', 0, 'B'),
('a', 1, 'A'),
('a', 1, 'B'),
('b', 0, 'A'),
('b', 0, 'B'),
('b', 1, 'A'),
('b', 1, 'B')]
Replicating behavior with regular Python¶
Note: Just as with lists, addition for tuples is concatenation.
In [27]:
('a', 0) + ('A',)
Out[27]:
('a', 0, 'A')
In [28]:
def product(*args):
if not args:
yield ()
else:
for items in product(*args[:-1]):
for item in args[-1]:
yield tuple(items) + (item,)
In [29]:
list(product('ab', range(2), 'AB'))
Out[29]:
[('a', 0, 'A'),
('a', 0, 'B'),
('a', 1, 'A'),
('a', 1, 'B'),
('b', 0, 'A'),
('b', 0, 'B'),
('b', 1, 'A'),
('b', 1, 'B')]
permuations¶
In [30]:
list(it.permutations('abc', 2))
Out[30]:
[('a', 'b'), ('a', 'c'), ('b', 'a'), ('b', 'c'), ('c', 'a'), ('c', 'b')]
In [31]:
list(it.permutations('abc'))
Out[31]:
[('a', 'b', 'c'),
('a', 'c', 'b'),
('b', 'a', 'c'),
('b', 'c', 'a'),
('c', 'a', 'b'),
('c', 'b', 'a')]
Replicating behavior with regular Python¶
In [32]:
def permutations(xs, n=None):
if n is None:
n = len(xs)
args = repeat(xs, n)
for item in product(*args):
if len(set(item)) == n:
yield item
In [33]:
list(it.permutations('abc', 2))
Out[33]:
[('a', 'b'), ('a', 'c'), ('b', 'a'), ('b', 'c'), ('c', 'a'), ('c', 'b')]
In [34]:
list(permutations('abc'))
Out[34]:
[('a', 'b', 'c'),
('a', 'c', 'b'),
('b', 'a', 'c'),
('b', 'c', 'a'),
('c', 'a', 'b'),
('c', 'b', 'a')]