If I've got an iterable containing strings, is there a simple way to turn it into a stream? I want to do something like this:
def make_file():
yield "hello\n"
yield "world\n"
output = tarfile.TarFile(…)
stream = iterable_to_stream(make_file())
output.addfile(…, stream)
Python 3 has a new I/O stream API (library docs), replacing the old file-like object protocol. (The new API is also available in Python 2 in the io module, and it's backwards-compatible with the file-like object protocol.)
Here's an implementation for the new API, in Python 2 and 3:
import io
def iterable_to_stream(iterable, buffer_size=io.DEFAULT_BUFFER_SIZE):
"""
Lets you use an iterable (e.g. a generator) that yields bytestrings as a read-only
input stream.
The stream implements Python 3's newer I/O API (available in Python 2's io module).
For efficiency, the stream is buffered.
"""
class IterStream(io.RawIOBase):
def __init__(self):
self.leftover = None
def readable(self):
return True
def readinto(self, b):
try:
l = len(b) # We're supposed to return at most this much
chunk = self.leftover or next(iterable)
output, self.leftover = chunk[:l], chunk[l:]
b[:len(output)] = output
return len(output)
except StopIteration:
return 0 # indicate EOF
return io.BufferedReader(IterStream(), buffer_size=buffer_size)
Example usage:
with iterable_to_stream(str(x**2).encode('utf8') for x in range(11)) as s:
print(s.read())
Since it doesn't look like there is a "standard" way of doing it, I've banged together a simple implementation:
class iter_to_stream(object):
def __init__(self, iterable):
self.buffered = ""
self.iter = iter(iterable)
def read(self, size):
result = ""
while size > 0:
data = self.buffered or next(self.iter, None)
self.buffered = ""
if data is None:
break
size -= len(data)
if size < 0:
data, self.buffered = data[:size], data[size:]
result += data
return result
Here's my streaming iterator an experimental branch of urllib3 supporting streaming chunked request via iterables:
class IterStreamer(object):
"""
File-like streaming iterator.
"""
def __init__(self, generator):
self.generator = generator
self.iterator = iter(generator)
self.leftover = ''
def __len__(self):
return self.generator.__len__()
def __iter__(self):
return self.iterator
def next(self):
return self.iterator.next()
def read(self, size):
data = self.leftover
count = len(self.leftover)
if count < size:
try:
while count < size:
chunk = self.next()
data += chunk
count += len(chunk)
except StopIteration:
pass
self.leftover = data[size:]
return data[:size]
Source with context: https://github.com/shazow/urllib3/blob/filepost-stream/urllib3/filepost.py#L23
Related unit tests: https://github.com/shazow/urllib3/blob/filepost-stream/test/test_filepost.py#L9
Alas this code hasn't made it into the stable branch yet as sizeless chunked requests are poorly supported, but it should be a good foundation for what you're trying to do. See the source link for examples showing how it can be used.
A starting point:
class iterable_to_stream:
def __init__(self, iterable):
self.iter = iter(iterable)
def read(self):
try:
return self.iter.next()
except StopIteration:
return ""
A bit modified version of a great Mechanical snail's answer. Here, readinto(b) implementation makes multiple calls to the the underlying iterator, in order to gather as much as possible amount of bytes for the size of the given writable bytes-like object b.
class IteratorReader(io.RawIOBase):
def __init__(self, iterator):
self.iterator = iterator
self.leftover = []
def readinto(self, buffer: bytearray) -> Optional[int]:
size = len(buffer)
while len(self.leftover) < size:
try:
self.leftover.extend(next(self.iterator))
except StopIteration:
break
if len(self.leftover) == 0:
return 0
output, self.leftover = self.leftover[:size], self.leftover[size:]
buffer[:len(output)] = output
return len(output)
def readable(self) -> bool:
return True
and usage:
def iterator1():
for i in ('a', 'b', 'c', 'd', 'e', 'f', 'g'):
res = i * 3
yield res.encode("utf8")
iterreader = IteratorReader(iterator1())
while True:
r = iterreader.read(4)
if not r:
break
print(r)
TarFile takes anything that provides a file-like interface -- so you could either use StringIO (io.StringIO if you are using Python 3.X) to yield what you need to TarFile.addfile() or you could create your own class that provides a file-like interface and yields what you need.
