Python

This is a code snippet for streaming gzipped text data from web in python 3.

import urllib.request
import gzip
import io
import itertools

url = "https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2019-39/segments/1568514570740.10/wet/CC-MAIN-20190915052433-20190915074433-00000.warc.wet.gz"
conn = io.TextIOWrapper(gzip.GzipFile(fileobj=urllib.request.urlopen(url)))

for line in itertools.islice(conn, 10):
    print(line.rstrip())