import hyper
c = hyper.HTTP20Connection('twitter.com')
c.request('GET', '/')
r = c.getresponse()
print(r.status)
r.close()

import http.client as http
c = http.HTTPSConnection('twitter.com')
c.request('GET', '/')
r = c.getresponse()
print(r.status)
r.close()

import time

class Timer(object):
    def __init__(self):
        self.start = None
        self.end = None
        self.interval = None
        
    def __enter__(self):
        self.start = time.time()
        return self
    
    def __exit__(self, *args):
        self.end = time.time()
        self.interval = self.end - self.start

c1 = http.HTTPSConnection('twitter.com')
c2 = hyper.HTTP20Connection('twitter.com')

with Timer() as t1:
    c1.request('GET', '/')
    r1 = c1.getresponse()
    d1 = r1.read()
    
with Timer() as t2:
    c2.request('GET', '/')
    r2 = c2.getresponse()
    d2 = r2.read()
    
c1.close()
c2.close()

print("HTTP/1.1 total time: {:.3f}".format(t1.interval))
print("HTTP/2   total time: {:.3f}".format(t2.interval))

headers = {'Accept': '*/*', 'Accept-Encoding': 'gzip, deflate', 'User-Agent': 'python-requests/2.2.1 CPython/3.4.1 Windows/7'}

c1 = http.HTTPSConnection('twitter.com')
c2 = hyper.HTTP20Connection('twitter.com')

with Timer() as t1:
    c1.request('GET', '/', headers=headers)
    r1 = c1.getresponse()
    d1 = r1.read()
    
with Timer() as t2:
    c2.request('GET', '/', headers=headers)
    r2 = c2.getresponse()
    d2 = r2.read()
    
c1.close()
c2.close()

print("HTTP/1.1 total time: {:.3f}".format(t1.interval))
print("HTTP/2   total time: {:.3f}".format(t2.interval))

import collections
import re
import itertools

ABSOLUTE_URL_RE = re.compile(b'<a href="https://nghttp2.org(/\\S+)"')
RELATIVE_URL_RE = re.compile(b'<a href="(/[^/]\\S+)"')

def get_urls(html):
    # We're going to get nghttp2 urls out of the text by regular expression.
    # This doesn't work in the general case, but this is a toy implementation
    # so it'll be fine.
    absolute_urls = ABSOLUTE_URL_RE.finditer(html)
    relative_urls = RELATIVE_URL_RE.finditer(html)
    return list(u.group(1) for u in itertools.chain(absolute_urls, relative_urls))

def http1_scrape():
    # We removed Accept-Encoding from the headers because httplib doesn't support gzip or deflate by default. HTTP/2
    # makes supporting gzip mandatory: be aware.
    headers = {'Accept': '*/*', 'User-Agent': 'python-httplib/3.4.1 CPython/3.4.1 Windows/7'}
    visit_queue = collections.deque(['/'])
    seen = set(['/'])
    
    conn = http.HTTPSConnection('nghttp2.org')
    
    while visit_queue:
        url = visit_queue.popleft()
        
        conn.request('GET', url, headers=headers)
        r = conn.getresponse()
        html = r.read()
        
        found_paths = get_urls(html)
        
        for path in found_paths:
            # Canonicalise the path.
            path = path.decode('utf-8')
            path = path.rstrip('/')
            if path not in seen:
                seen.add(path)
                visit_queue.append(path)

    return len(seen)
                
# Start the scrape and time it.
with Timer() as t:
    count = http1_scrape()

print("HTTP/1.1 scrape took {:.3f} seconds to scrape {:d} URLs".format(t.interval, count))

def http2_scrape():
    headers = {'Accept': '*/*', 'User-Agent': 'python-httplib/3.4.1 CPython/3.4.1 Windows/7'}
    ids = collections.deque()
    seen = set(['/'])
    
    # Set the first request off
    conn = hyper.HTTP20Connection('nghttp2.org')
    stream_id = conn.request('GET', '/', headers=headers)
    ids.append(stream_id)
    
    while ids:
        stream_id = ids.popleft()
        r = conn.getresponse(stream_id)
        html = r.read()
        
        found_paths = get_urls(html)
        
        for path in found_paths:
            # Canonicalise the path.
            path = path.decode('utf-8')
            path = path.rstrip('/')
            if path not in seen:
                seen.add(path)
                stream_id = conn.request('GET', path, headers=headers)
                ids.append(stream_id)

    return len(seen)

# Start the scrape and time it.
with Timer() as t:
    count = http2_scrape()
    
print("HTTP/2 scrape took {:.3f} seconds to scrape {:d} URLs".format(t.interval, count))

import requests

RELATIVE_URL_RE = re.compile('<a href="(/[^/]\\S+)"')

def get_urls(html):
    # We don't need the absolute URL regex because Twitter
    # simply doesn't use them.
    relative_urls = RELATIVE_URL_RE.finditer(html)
    return list(u.group(1) for u in relative_urls)

def http1_requests_scrape():
    # The termination condition this time around is 50 pages.
    # Let's do a random walk.
    visit_queue = collections.deque(['/'])
    s = requests.Session()
    seen = set(['/'])
    
    for _ in range(50):
        url = visit_queue.popleft()
        url = 'https://twitter.com' + url
        
        r = s.get(url)
        found_paths = get_urls(r.text)
        
        for path in found_paths:
            # Canonicalise the path.
            path = path.rstrip('/')
            if path not in seen:
                seen.add(path)
                visit_queue.append(path)

    return len(seen)

# Start the scrape and time it.
with Timer() as t:
    count = http1_requests_scrape()

print("HTTP/1.1 scrape took {:.3f} seconds to scrape 50 URLs".format(t.interval, count))

from hyper.contrib import HTTP20Adapter

def http2_requests_scrape():
    # The termination condition this time around is 50 pages.
    # Let's do a random walk.
    visit_queue = collections.deque(['/'])
    s = requests.Session()
    
    # Note that these three lines are the only difference between
    # the two Requests examples
    a = HTTP20Adapter()
    s.mount('https://twitter.com', a)
    s.mount('https://www.twitter.com', a) # To ensure that redirects use the adapter.
    seen = set(['/'])
    
    for _ in range(50):
        url = visit_queue.popleft()
        url = 'https://twitter.com' + url
        
        r = s.get(url)
        found_paths = get_urls(r.text)
        
        for path in found_paths:
            # Canonicalise the path.
            path = path.rstrip('/')
            if path not in seen:
                seen.add(path)
                visit_queue.append(path)

    return len(seen)

# Start the scrape and time it.
with Timer() as t:
    count = http2_requests_scrape()

print("HTTP/2 scrape took {:.3f} seconds to scrape 50 URLs".format(t.interval, count))

import concurrent.futures

def time_execution_of(func):
    with Timer() as t:
        func()
    return t.interval

http1_funcs = [http1_requests_scrape] * 50
http2_funcs = [http2_requests_scrape] * 50
    
with concurrent.futures.ProcessPoolExecutor() as e:
    print("HTTP/1.1 scrape takes, on average, {:.3f} seconds to scrape 50 URLs".format(
        sum(e.map(time_execution_of, http1_funcs)) / 50
    ))
    
with concurrent.futures.ProcessPoolExecutor() as e:
    print("HTTP/2   scrape takes, on average, {:.3f} seconds to scrape 50 URLs".format(
        sum(e.map(time_execution_of, http2_funcs)) / 50
    ))