#!/usr/bin/env python
# coding: utf-8
# tqdm
# 
#
# [](https://pypi.org/project/tqdm)|[](https://tqdm.github.io/releases)|[](https://anaconda.org/conda-forge/tqdm)|[](https://hub.docker.com/r/tqdm/tqdm)|[](https://snapcraft.io/tqdm)
# -|-|-|-|-
#
# [](https://github.com/tqdm/tqdm/actions/workflows/test.yml)|[](https://coveralls.io/github/tqdm/tqdm)|[](https://codecov.io/gh/tqdm/tqdm)|[](https://www.codacy.com/gh/tqdm/tqdm/dashboard)|[](https://libraries.io/pypi/tqdm)|[](https://pepy.tech/project/tqdm)
# -|-|-|-|-|-
#
# [](https://doi.org/10.5281/zenodo.595120)|[](https://raw.githubusercontent.com/tqdm/tqdm/master/LICENCE)|[](https://www.openhub.net/p/tqdm?ref=Thin+badge)|[](https://mybinder.org/v2/gh/tqdm/tqdm/master?filepath=DEMO.ipynb)|[](https://github.com/vinta/awesome-python)
# -|-|-|-|-
#
# `tqdm` derives from the Arabic word *taqaddum* (تقدّم) which can mean
# "progress," and is an abbreviation for "I love you so much" in Spanish
# (*te quiero demasiado*).
#
# Instantly make your loops show a smart progress meter - just wrap any
# iterable with `tqdm(iterable)`, and you're done!
# In[ ]:
from tqdm import tqdm
for i in tqdm(range(10000)):
pass
# `trange(N)` can be also used as a convenient shortcut for
# `tqdm(range(N))`.
# In[ ]:
from tqdm import trange
for i in trange(10000, unit_scale=True, desc="hello", unit="epoch"):
pass
# |[](https://tqdm.github.io/video) [](https://tqdm.github.io/PyData2019/slides.html) [](https://tqdm.github.io/merch)
# -|-
#
# It can also be executed as a module with pipes:
# In[ ]:
get_ipython().system(' seq 9999999 | tqdm --bytes | wc -l')
# ```sh
# tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` > backup.tgz
# 44%|██████████████▊ | 153M/352M [00:14<00:18, 11.0MB/s]
# ```
# Overhead is low -- about 60ns per iteration (80ns with `tqdm.gui`), and
# is unit tested against performance regression. By comparison, the
# well-established
# [ProgressBar](https://github.com/niltonvolpato/python-progressbar) has
# an 800ns/iter overhead.
#
# In addition to its low overhead, `tqdm` uses smart algorithms to predict
# the remaining time and to skip unnecessary iteration displays, which
# allows for a negligible overhead in most cases.
#
# `tqdm` works on any platform (Linux, Windows, Mac, FreeBSD, NetBSD,
# Solaris/SunOS), in any console or in a GUI, and is also friendly with
# IPython/Jupyter notebooks.
#
# `tqdm` does not require any dependencies (not even `curses`!), just
# Python and an environment supporting `carriage return \r` and
# `line feed \n` control characters.
#
# ---
#
# ## Usage
#
# `tqdm` is very versatile and can be used in a number of ways.
# The three main ones are given below.
#
# ### Iterable-based
#
# Wrap `tqdm()` around any iterable:
# In[ ]:
from tqdm import tqdm
from time import sleep
# In[ ]:
text = ""
for char in tqdm(["a", "b", "c", "d"]):
sleep(0.25)
text = text + char
# `trange(i)` is a special optimised instance of `tqdm(range(i))`:
# In[ ]:
from tqdm import trange
for i in trange(100):
sleep(0.01)
# Instantiation outside of the loop allows for manual control over `tqdm()`:
# In[ ]:
pbar = tqdm(["a", "b", "c", "d"])
for char in pbar:
sleep(0.25)
pbar.set_description("Processing %s" % char)
# ### Manual
#
# Manual control of `tqdm()` updates using a `with` statement:
# In[ ]:
with tqdm(total=100) as pbar:
for i in range(10):
sleep(0.1)
pbar.update(10)
# If the optional variable `total` (or an iterable with `len()`) is
# provided, predictive stats are displayed.
#
# `with` is also optional (you can just assign `tqdm()` to a variable,
# but in this case don't forget to `del` or `close()` at the end:
# In[ ]:
pbar = tqdm(total=100)
for i in range(10):
sleep(0.1)
pbar.update(10)
pbar.close()
# ### Module
# Perhaps the most wonderful use of `tqdm` is in a script or on the
# command line. Simply inserting `tqdm` (or `python -m tqdm`) between
# pipes will pass through all `stdin` to `stdout` while printing progress
# to `stderr`.
#
# The example below demonstrated counting the number of lines in all
# Python files in the current directory, with timing information included.
# In[ ]:
get_ipython().system(" time find . -name '*.py' -type f -exec cat \\{} \\; | wc -l")
# In[ ]:
get_ipython().system(" time find . -name '*.py' -type f -exec cat \\{} \\; | tqdm | wc -l")
# Note that the usual arguments for `tqdm` can also be specified.
# In[ ]:
get_ipython().system(" find . -name '*.py' -type f -exec cat \\{} \\; | tqdm --unit loc --unit-scale --total 4104300 --null")
# Backing up a large directory?
#
# ```sh
# tar -zcf - docs/ | tqdm --bytes --total `du -sb docs/ | cut -f1` > backup.tgz
# 44%|██████████████▊ | 153M/352M [00:14<00:18, 11.0MB/s]
# ```
#
# This can be beautified further:
#
# ```sh
# BYTES="$(du -sb docs/ | cut -f1)"
# tar -cf - docs/ \
# | tqdm --bytes --total "$BYTES" --desc Processing | gzip \
# | tqdm --bytes --total "$BYTES" --desc Compressed --position 1 \
# > ~/backup.tgz
# Processing: 100%|██████████████████████| 352M/352M [00:14<00:00, 30.2MB/s]
# Compressed: 42%|█████████▎ | 148M/352M [00:14<00:19, 10.9MB/s]
# ```
#
# Or done on a file level using 7-zip:
#
# ```sh
# 7z a -bd -r backup.7z docs/ | grep Compressing \
# | tqdm --total $(find docs/ -type f | wc -l) --unit files \
# | grep -v Compressing
# 100%|██████████████████████████▉| 15327/15327 [01:00<00:00, 712.96files/s]
# ```
# ## Documentation
# In[ ]:
get_ipython().run_line_magic('pinfo', 'tqdm')
# In[ ]:
get_ipython().system(' tqdm --help')
# ## Examples and Advance Usage
#
# - See the [examples](https://github.com/tqdm/tqdm/tree/master/examples)
# folder;
# - import the module and run `help()`;
# - consult the [wiki](https://github.com/tqdm/tqdm/wiki)
# - this has an
# [excellent article](https://github.com/tqdm/tqdm/wiki/How-to-make-a-great-Progress-Bar)
# on how to make a **great** progressbar;
# - check out the [slides from PyData London](https://tqdm.github.io/PyData2019/slides.html), or
# - run this file!
#
# ### Description and additional stats
#
# Custom information can be displayed and updated dynamically on `tqdm` bars
# with the `desc` and `postfix` arguments:
# In[ ]:
from tqdm import tqdm, trange
from random import random, randint
from time import sleep
with trange(10) as t:
for i in t:
# Description will be displayed on the left
t.set_description('GEN %i' % i)
# Postfix will be displayed on the right,
# formatted automatically based on argument's datatype
t.set_postfix(loss=random(), gen=randint(1,999), str='h',
lst=[1, 2])
sleep(0.1)
with tqdm(total=10, bar_format="{postfix[0]} {postfix[1][value]:>8.2g}",
postfix=["Batch", dict(value=0)]) as t:
for i in range(10):
sleep(0.1)
t.postfix[1]["value"] = i / 2
t.update()
# Points to remember when using `{postfix[...]}` in the `bar_format` string:
#
# - `postfix` also needs to be passed as an initial argument in a
# compatible format, and
# - `postfix` will be auto-converted to a string if it is a `dict`-like
# object. To prevent this behaviour, insert an extra item into the
# dictionary where the key is not a string.
#
# Additional `bar_format` parameters may also be defined by overriding
# `format_dict`, and the bar itself may be modified using `ascii`:
# In[ ]:
from tqdm import tqdm
from time import sleep
class TqdmExtraFormat(tqdm):
"""Provides a `total_time` format parameter"""
@property
def format_dict(self):
d = super().format_dict
total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1)
d.update(total_time=self.format_interval(total_time) + " in total")
return d
for i in TqdmExtraFormat(
range(9), ascii=" .oO0",
bar_format="{total_time}: {percentage:.0f}%|{bar}{r_bar}"):
if i == 4:
break
# Note that `{bar}` also supports a format specifier `[width][type]`.
#
# - `width`
# + unspecified (default): automatic to fill `ncols`
# + `int >= 0`: fixed width overriding `ncols` logic
# + `int < 0`: subtract from the automatic default
# - `type`
# + `a`: ascii (`ascii=True` override)
# + `u`: unicode (`ascii=False` override)
# + `b`: blank (`ascii=" "` override)
#
# This means a fixed bar with right-justified text may be created by
# using: `bar_format="{l_bar}{bar:10}|{bar:-10b}right-justified"`
# ### Nested progress bars
#
# `tqdm` supports nested progress bars. Here's an example:
# In[ ]:
from tqdm.auto import trange
from time import sleep
for i in trange(4, desc='1st loop'):
for j in trange(5, desc='2nd loop'):
for k in trange(50, desc='3rd loop', leave=False):
sleep(0.01)
# For manual control over positioning (e.g. for multi-processing use),
# you may specify `position=n` where `n=0` for the outermost bar, `n=1`
# for the next, and so on. However, it's best to check if tqdm can work
# without manual position first.
#
# ```python
# from time import sleep
# from tqdm import trange, tqdm
# from multiprocessing import Pool, RLock, freeze_support
#
# L = list(range(9))
#
# def progresser(n):
# interval = 0.001 / (n + 2)
# total = 5000
# text = "#{}, est. {:<04.2}s".format(n, interval * total)
# for _ in trange(total, desc=text, position=n):
# sleep(interval)
#
# if __name__ == '__main__':
# freeze_support() # for Windows support
# tqdm.set_lock(RLock()) # for managing output contention
# p = Pool(initializer=tqdm.set_lock, initargs=(tqdm.get_lock(),))
# p.map(progresser, L)
# ```
#
# Note that in Python 3, `tqdm.write` is thread-safe:
#
# ```python
# from time import sleep
# from tqdm import tqdm, trange
# from concurrent.futures import ThreadPoolExecutor
#
# L = list(range(9))
#
# def progresser(n):
# interval = 0.001 / (n + 2)
# total = 5000
# text = "#{}, est. {:<04.2}s".format(n, interval * total)
# for _ in trange(total, desc=text):
# sleep(interval).auto
# if n == 6:
# tqdm.write("n == 6 completed.")
# tqdm.write("`tqdm.write()` is thread-safe in py3!")
#
# if __name__ == '__main__':
# with ThreadPoolExecutor() as p:
# p.map(progresser, L)
# ```
# ### Hooks and callbacks
#
# `tqdm` can easily support callbacks/hooks and manual updates.
# Here's an example with `urllib`:
#
# **`urllib.urlretrieve` documentation**
#
# > [...]
# > If present, the hook function will be called once
# > on establishment of the network connection and once after each block read
# > thereafter. The hook will be passed three arguments; a count of blocks
# > transferred so far, a block size in bytes, and the total size of the file.
# > [...]
# In[ ]:
import urllib, os
from tqdm import tqdm
urllib = getattr(urllib, 'request', urllib)
class TqdmUpTo(tqdm):
"""Provides `update_to(n)` which uses `tqdm.update(delta_n)`."""
def update_to(self, b=1, bsize=1, tsize=None):
"""
b : int, optional
Number of blocks transferred so far [default: 1].
bsize : int, optional
Size of each block (in tqdm units) [default: 1].
tsize : int, optional
Total size (in tqdm units). If [default: None] remains unchanged.
"""
if tsize is not None:
self.total = tsize
return self.update(b * bsize - self.n) # also sets self.n = b * bsize
eg_link = "https://caspersci.uk.to/matryoshka.zip"
with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1,
desc=eg_link.split('/')[-1]) as t: # all optional kwargs
urllib.urlretrieve(eg_link, filename=os.devnull,
reporthook=t.update_to, data=None)
t.total = t.n
# Inspired by [twine#242](https://github.com/pypa/twine/pull/242).
# Functional alternative in
# [examples/tqdm_wget.py](https://github.com/tqdm/tqdm/blob/master/examples/tqdm_wget.py).
#
# It is recommend to use `miniters=1` whenever there is potentially large
# differences in iteration speed (e.g. downloading a file over a patchy
# connection).
#
# **Wrapping read/write methods**
#
# To measure throughput through a file-like object's `read` or `write`
# methods, use `CallbackIOWrapper`:
#
# ```python
# from tqdm import tqdm
# from tqdm.utils import CallbackIOWrapper
#
# with tqdm(total=file_obj.size,
# unit='B', unit_scale=True, unit_divisor=1024) as t:
# fobj = CallbackIOWrapper(t.update, file_obj, "read")
# while True:
# chunk = fobj.read(chunk_size)
# if not chunk:
# break
# t.reset()
# # ... continue to use `t` for something else
# ```
#
# Alternatively, use the even simpler `wrapattr` convenience function,
# which would condense both the `urllib` and `CallbackIOWrapper` examples
# down to:
# In[ ]:
import urllib, os
from tqdm import tqdm
eg_link = "https://caspersci.uk.to/matryoshka.zip"
response = getattr(urllib, 'request', urllib).urlopen(eg_link)
with tqdm.wrapattr(open(os.devnull, "wb"), "write",
miniters=1, desc=eg_link.split('/')[-1],
total=getattr(response, 'length', None)) as fout:
for chunk in response:
fout.write(chunk)
# The `requests` equivalent is nearly identical:
# In[ ]:
import requests, os
from tqdm import tqdm
eg_link = "https://caspersci.uk.to/matryoshka.zip"
response = requests.get(eg_link, stream=True)
with tqdm.wrapattr(open(os.devnull, "wb"), "write",
miniters=1, desc=eg_link.split('/')[-1],
total=int(response.headers.get('content-length', 0))) as fout:
for chunk in response.iter_content(chunk_size=4096):
fout.write(chunk)
# ### Pandas Integration
#
# Due to popular demand we've added support for `pandas` -- here's an example
# for `DataFrame.progress_apply` and `DataFrameGroupBy.progress_apply`:
# In[ ]:
import pandas as pd
import numpy as np
from tqdm import tqdm
df = pd.DataFrame(np.random.randint(0, 100, (100000, 6)))
# Register `pandas.progress_apply` and `pandas.Series.map_apply` with `tqdm`
# (can use `tqdm.gui.tqdm`, `tqdm.notebook.tqdm`, optional kwargs, etc.)
tqdm.pandas(desc="my bar!")
# Now you can use `progress_apply` instead of `apply`
# and `progress_map` instead of `map`
df.progress_apply(lambda x: x**2)
# can also groupby:
# df.groupby(0).progress_apply(lambda x: x**2)
# In case you're interested in how this works (and how to modify it for
# your own callbacks), see the
# [examples](https://github.com/tqdm/tqdm/tree/master/examples) folder or
# import the module and run `help()`.
#
# ### Keras Integration
#
# A `keras` callback is also available:
#
# ```python
# from tqdm.keras import TqdmCallback
#
# ...
#
# model.fit(..., verbose=0, callbacks=[TqdmCallback()])
# ```
#
# ### IPython/Jupyter Integration
#
# IPython/Jupyter is supported via the `tqdm.notebook` submodule:
# In[ ]:
from tqdm.notebook import trange, tqdm
from time import sleep
for i in trange(3, desc='1st loop'):
for j in tqdm(range(100), desc='2nd loop'):
sleep(0.01)
# In addition to `tqdm` features, the submodule provides a native Jupyter
# widget (compatible with IPython v1-v4 and Jupyter), fully working nested
# bars and colour hints (blue: normal, green: completed, red:
# error/interrupt, light blue: no ETA); as demonstrated below.
#
# 
#
# The `notebook` version supports percentage or pixels for overall width
# (e.g.: `ncols='100%'` or `ncols='480px'`).
#
# It is also possible to let `tqdm` automatically choose between console
# or notebook versions by using the `autonotebook` submodule:
# In[ ]:
from tqdm.autonotebook import tqdm
tqdm.pandas()
# Note that this will issue a `TqdmExperimentalWarning` if run in a
# notebook since it is not meant to be possible to distinguish between
# `jupyter notebook` and `jupyter console`. Use `auto` instead of
# `autonotebook` to suppress this warning.
#
# Note that notebooks will display the bar in the cell where it was
# created. This may be a different cell from the one where it is used. If
# this is not desired, the creation of the bar must be delayed/moved to
# the cell where it is desired to be displayed.
#
# Another possibility is to have a single bar (near the top of the
# notebook) which is constantly re-used (using `reset()` rather than
# `close()`). For this reason, the notebook version (unlike the CLI
# version) does not automatically call `close()` upon `Exception`.
# In[ ]:
from tqdm.notebook import tqdm
pbar = tqdm()
# In[ ]:
# different cell
iterable = range(100)
pbar.reset(total=len(iterable)) # initialise with new `total`
for i in iterable:
pbar.update()
pbar.refresh() # force print final status but don't `close()`
# ### Writing messages
#
# This is a work in progress (see
# [#737](https://github.com/tqdm/tqdm/issues/737)).
#
# Since `tqdm` uses a simple printing mechanism to display progress bars,
# you should not write any message in the terminal using `print()` while a
# progressbar is open.
#
# To write messages in the terminal without any collision with `tqdm` bar
# display, a `.write()` method is provided:
# In[ ]:
from tqdm.auto import tqdm, trange
from time import sleep
bar = trange(10)
for i in bar:
# Print using tqdm class method .write()
sleep(0.1)
if not (i % 3):
tqdm.write("Done task %i" % i)
# Can also use bar.write()
# By default, this will print to standard output `sys.stdout`. but you can
# specify any file-like object using the `file` argument. For example,
# this can be used to redirect the messages writing to a log file or class.
#
# [](https://caspersci.uk.to/cgi-bin/hits.cgi?q=tqdm&a=plot&r=https://github.com/tqdm/tqdm&l=https://tqdm.github.io/img/favicon.png&f=https://tqdm.github.io/img/logo.gif&style=social)|(Since 19 May 2016)
# -|-
# ## Do your own experiments here 👇
#
# Try `tqdm` youself by adding your code below and running your own experiments.
# In[ ]:
import tqdm
# your code here
tqdm.