Fetch and show a summary of conda-forge's travis queue
import os
import yaml
with open(os.path.expanduser('~/.travis/config.yml')) as f:
cfg = yaml.safe_load(f)
token = cfg['endpoints']['https://api.travis-ci.org/']['access_token']
import os
import asyncio
from asyncio import get_event_loop, Semaphore
import aiohttp
from collections import defaultdict
loop = get_event_loop()
all_jobs = defaultdict(lambda : [])
all_builds = defaultdict(lambda : [])
headers = {
'Travis-API-Version': '3',
'Accept': 'application/json',
'Authorization': f'token {token}',
}
TRAVIS = "https://api.travis-ci.org"
# limit concurrent requests to travis API
sem = Semaphore(20)
async def list_repos(url):
futures = []
async with aiohttp.ClientSession() as session:
async with sem, session.get(url, headers=headers) as response:
data = await response.json()
for repo in data['repositories']:
if repo['active']:
futures.append(asyncio.ensure_future(list_builds(repo, sem)))
finished = [ ]
so_far = data['@pagination']['offset'] + data['@pagination']['limit']
total = data['@pagination']['count']
if data['@pagination']['is_last']:
so_far = total
print(f"{so_far}/{total} repos")
if not data['@pagination']['is_last']:
next_url = f"{TRAVIS}{data['@pagination']['next']['@href']}"
next_f = asyncio.ensure_future(list_repos(next_url))
else:
next_f = None
await asyncio.gather(*futures)
print(f"{len(all_builds)} builds, {len(all_jobs)} jobs")
if next_f:
await next_f
async def list_builds(repo, sem):
url_repo = f'{TRAVIS}/repo/{repo["id"]}/builds?state=created'
if repo['name'] in all_jobs:
# short-circuit re-runs
return
async with sem, aiohttp.ClientSession() as session:
data_build = await session.get(url_repo, headers=headers)
builds = await data_build.json()
builds = builds['builds']
repo_name = repo['name']
futures = []
for build in builds:
all_builds[repo_name].append(build)
for job in build['jobs']:
futures.append(asyncio.ensure_future(get_job(job, sem)))
await asyncio.gather(*futures)
async def get_job(job_info, sem):
url = f"{TRAVIS}{job_info['@href']}"
async with sem, aiohttp.ClientSession() as session:
r = await session.get(url, headers=headers)
job = await r.json()
if job['state'] == 'canceled':
# omit canceled jobs
return
all_jobs[job['repository']['name']].append(job)
def get_all_builds(owner):
return list_repos(f"{TRAVIS}/owner/{owner}/repos")
%%time
loop.run_until_complete(get_all_builds('conda-forge'))
100/3844 repos 200/3844 repos 300/3844 repos 85 builds, 82 jobs 96 builds, 86 jobs 96 builds, 90 jobs 400/3844 repos 107 builds, 100 jobs 500/3844 repos 109 builds, 101 jobs 600/3844 repos 113 builds, 105 jobs 700/3844 repos 114 builds, 106 jobs 800/3844 repos 900/3844 repos 121 builds, 112 jobs 1000/3844 repos 124 builds, 115 jobs 1100/3844 repos 128 builds, 115 jobs 1200/3844 repos 128 builds, 115 jobs 1300/3844 repos 130 builds, 117 jobs 1400/3844 repos 133 builds, 119 jobs 1500/3844 repos 135 builds, 119 jobs 1600/3844 repos 136 builds, 120 jobs 1700/3844 repos 136 builds, 120 jobs 1800/3844 repos 138 builds, 122 jobs 1900/3844 repos 139 builds, 122 jobs 139 builds, 122 jobs 2000/3844 repos 141 builds, 123 jobs 2100/3844 repos 143 builds, 123 jobs 2200/3844 repos 143 builds, 123 jobs 2300/3844 repos 144 builds, 124 jobs 2400/3844 repos 144 builds, 124 jobs 2500/3844 repos 145 builds, 124 jobs 2600/3844 repos 145 builds, 124 jobs 2700/3844 repos 146 builds, 124 jobs 2800/3844 repos 149 builds, 126 jobs 2900/3844 repos 150 builds, 126 jobs 3000/3844 repos 151 builds, 126 jobs 3100/3844 repos 152 builds, 126 jobs 3200/3844 repos 152 builds, 126 jobs 3300/3844 repos 153 builds, 127 jobs 3400/3844 repos 157 builds, 128 jobs 3500/3844 repos 159 builds, 129 jobs 3600/3844 repos 162 builds, 129 jobs 3700/3844 repos 162 builds, 129 jobs 3800/3844 repos 163 builds, 129 jobs 3844/3844 repos 163 builds, 129 jobs CPU times: user 1min 19s, sys: 4.61 s, total: 1min 23s Wall time: 4min 3s
Get all builds, sorted by created_at
from itertools import chain
from operator import itemgetter
def get_date(job):
"""get date for sorting
Use updated_at for 'created' jobs,
which includes resubmit info.
This won't be correct once the job has started,
so fallback on 'created' date for those.
"""
if job['state'] == 'created':
return job['updated_at']
else:
return job['created_at']
ordered_jobs = sorted(chain(*all_jobs.values()),
key=get_date)
Show a view of the current queue
from dateutil.parser import parse as parse_date
from datetime import datetime, timedelta, timezone
now = datetime.now().astimezone(timezone.utc)
all_authors = set(build['created_by']['login'] for build in chain(*all_builds.values()))
lines = [
f"## Conda-Forge travis queue status as of {now.strftime('%Y-%m-%d %H:%M UTC')}",
"",
f"{len(all_builds)} builds and {len(all_jobs)} jobs",
"",
f"Total unique authors: {len(all_authors)}",
"",
]
i = 1
for i, job in enumerate(ordered_jobs):
repo = job['repository']['slug']
url = f"https://travis-ci.org/{repo}/jobs/{job['id']}"
date = parse_date(get_date(job))
ago = now - date
if ago < timedelta(hours=24):
ago_s = f"{ago.total_seconds() / 3600:.1f} hours"
else:
ago_s = f"{ago.days} days"
s = f"{i}. {ago_s} ago [{repo}#{job['number']}]({url})"
if job['state'] != 'created':
s+= f" {job['state']}"
lines.append(s)
from IPython.display import Markdown
Markdown('\n'.join(lines))
163 builds and 129 jobs
Total unique authors: 95