%run scrap_IK_tweets.py
To read more about Document Object Model: https://developer.mozilla.org/en-US/docs/Web/API/Document_Object_Model/Introduction
To read more about Google Chrome Developer Tools: https://developer.chrome.com/docs/devtools/
Books Scraping WebSite (Version 1): https://arifpucit.github.io/bss1
Books Scraping WebSite (Version 2): https://arifpucit.github.io/bss2
urllib
Package¶urllib
package in Python 3 is a collection of following Python modules used for working with Uniform Resource Locators:urllib.request
for opening and reading URLs, using variety of protocolsurllib.error
containing the exceptions raised by urllib.requesturllib.parse
for parsing URLsurllib.robotparser
for parsing robots.txt filesThe
urllib.request.urlopen()
, may return a URLError sayingSSL: CERTIFICATE_VERIFY_FAILED
. To handle this error set the_create_default_https_context
attribute ofssl
to_create_unverified_context
import urllib
print(dir(urllib))
['__builtins__', '__cached__', '__doc__', '__file__', '__loader__', '__name__', '__package__', '__path__', '__spec__', 'error', 'parse', 'request', 'response']
print(dir(urllib.request))
['AbstractBasicAuthHandler', 'AbstractDigestAuthHandler', 'AbstractHTTPHandler', 'BaseHandler', 'CacheFTPHandler', 'ContentTooShortError', 'DataHandler', 'FTPHandler', 'FancyURLopener', 'FileHandler', 'HTTPBasicAuthHandler', 'HTTPCookieProcessor', 'HTTPDefaultErrorHandler', 'HTTPDigestAuthHandler', 'HTTPError', 'HTTPErrorProcessor', 'HTTPHandler', 'HTTPPasswordMgr', 'HTTPPasswordMgrWithDefaultRealm', 'HTTPPasswordMgrWithPriorAuth', 'HTTPRedirectHandler', 'HTTPSHandler', 'MAXFTPCACHE', 'OpenerDirector', 'ProxyBasicAuthHandler', 'ProxyDigestAuthHandler', 'ProxyHandler', 'Request', 'URLError', 'URLopener', 'UnknownHandler', '__all__', '__builtins__', '__cached__', '__doc__', '__file__', '__loader__', '__name__', '__package__', '__spec__', '__version__', '_cut_port_re', '_ftperrors', '_get_proxies', '_get_proxy_settings', '_have_ssl', '_localhost', '_noheaders', '_opener', '_parse_proxy', '_proxy_bypass_macosx_sysconf', '_randombytes', '_safe_gethostbyname', '_splitattr', '_splithost', '_splitpasswd', '_splitport', '_splitquery', '_splittag', '_splittype', '_splituser', '_splitvalue', '_thishost', '_to_bytes', '_url_tempfiles', 'addclosehook', 'addinfourl', 'base64', 'bisect', 'build_opener', 'contextlib', 'email', 'ftpcache', 'ftperrors', 'ftpwrapper', 'getproxies', 'getproxies_environment', 'getproxies_macosx_sysconf', 'hashlib', 'http', 'install_opener', 'io', 'localhost', 'noheaders', 'os', 'parse_http_list', 'parse_keqv_list', 'pathname2url', 'posixpath', 'proxy_bypass', 'proxy_bypass_environment', 'proxy_bypass_macosx_sysconf', 'quote', 're', 'request_host', 'socket', 'ssl', 'string', 'sys', 'tempfile', 'thishost', 'time', 'unquote', 'unquote_to_bytes', 'unwrap', 'url2pathname', 'urlcleanup', 'urljoin', 'urlopen', 'urlparse', 'urlretrieve', 'urlsplit', 'urlunparse', 'warnings']
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
resp = urllib.request.urlopen("https://arifpucit.github.io/bss2")
type(resp)
http.client.HTTPResponse
print(dir(resp))
['__abstractmethods__', '__class__', '__del__', '__delattr__', '__dict__', '__dir__', '__doc__', '__enter__', '__eq__', '__exit__', '__format__', '__ge__', '__getattribute__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__next__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '_abc_impl', '_checkClosed', '_checkReadable', '_checkSeekable', '_checkWritable', '_check_close', '_close_conn', '_get_chunk_left', '_method', '_peek_chunked', '_read1_chunked', '_read_and_discard_trailer', '_read_next_chunk_size', '_read_status', '_readall_chunked', '_readinto_chunked', '_safe_read', '_safe_readinto', 'begin', 'chunk_left', 'chunked', 'close', 'closed', 'code', 'debuglevel', 'detach', 'fileno', 'flush', 'fp', 'getcode', 'getheader', 'getheaders', 'geturl', 'headers', 'info', 'isatty', 'isclosed', 'length', 'msg', 'peek', 'read', 'read1', 'readable', 'readinto', 'readinto1', 'readline', 'readlines', 'reason', 'seek', 'seekable', 'status', 'tell', 'truncate', 'url', 'version', 'will_close', 'writable', 'write', 'writelines']
resp.code
200
html = resp.read()
print(html)
b'<!doctype html>\n<html lang="en">\n <head>\n <meta charset="utf-8">\n <meta name="viewport" content="width=device-width, initial-scale=1">\n <title>BSS2</title>\n <!-- external style sheet -->\n <link rel="stylesheet" href="./index.css">\n\n <!--Bootstrap style sheet-->\n <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-0evHe/X+R7YkIZDRvuzKMRqM+OrBnVFBL6DOitfPri4tjfHxaWutUpFmBp4vmVor" crossorigin="anonymous">\n\n <!--for icons of tick and cross ans star for in stock-->\n <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css"> \n\n </head>\n <body>\n <header class="header d-flex align-items-center justify-content-between">\n <img class="image-container" src="./images//arif.jpg" alt="arif"/>\n <p> <span class="large_text">Books Scraping Site</span></p>\n <img class="image-container" src="./images/pucit.jpg" alt="pucit"/>\n </header>\n <section>\n <div class="main-container d-flex align-items-start justify-content-between">\n <div class="navbar">\n <ul class="nav-links">\n <div class="link text-center" id="book_title">Books Titles</div>\n <li class="link book_type"><a href="index.html">Operating System</a></li>\n <li class="link book_type"><a href="SP.html">System Programming</a></li>\n <li class="link book_type"><a href="CA.html">Computer Architecture</a></li>\n </ul>\n </div>\n <div class="items">\n <div id="book_page_titile">Operating Systems</div>\n <div class="row">\n <div class="book_container col-sm-4">\n <img src="images/OS concepts.jpg" alt="" title="The Linux Programming Interface (TLPI) is the definitive guide to the Linux and UNIX programming interface\xe2\x80\x94the interface employed by nearly every application that runs on a Linux or UNIX system.">\n <p class="book_name"><a href="https://www.amazon.com/Operating-System-Concepts-Abridged-Companion/dp/1119456339" target="_blank">Operating System Concepts </br>By Avi Silberschatz</a></p>\n <div class="align-left">\n <p class=" price green">Rs.2000</p>\n <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p>\n <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div>\n <p class="review green" data-rating="20">20 Reviews</p>\n <button>Add to cart</button>\n </div>\n </div>\n <div class="book_container col-sm-4">\n <img src="images/Unix.jpg" alt="" title="It covers many topics not covered in older, more traditional textbook approaches, such as Python, UNIX System Programming from basics to socket-based network programming using the client-server paradigm, the Zettabyte File System (ZFS), and the highly developed X Windows-based KDE and Gnome GUI desktop environments">\n <p class="book_name"><a href="https://www.google.com/search?q=Unix+the+textbook+by+mansoor&rlz=1C1CHBD_enPK987PK987&oq=unix+the+textbook+by+mansoor&aqs=chrome.0.69i59j69i57j69i59j69i60l5.4419j0j7&sourceid=chrome&ie=UTF-8" target="_blank">UNIX The Textbook</br> By Syed Mansoor Sarwar</a></p>\n <div class="align-left">\n <p class=" price green">Rs.5000</p>\n <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p>\n <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span></div>\n <p class="review green" data-rating="100">100 Reviews</p>\n <button>Add to cart</button>\n </div>\n </div>\n <div class="book_container col-sm-4">\n <img src="images/taxonomy.jpg" alt="" title="Intrusion Detection using System Call Sequence">\n <p class="book_name"><a href="https://www.amazon.in/Taxonomy-Ids-Arif-Butt/dp/3639294092" target="_blank">Taxonomy of IDS </br>By Arif Butt</a></p>\n <div class="align-left">\n <p class=" price green">Rs.6900</p>\n <p class="stock not_stock " data-stock="not in stock"><i class="fa fa-times" aria-hidden="true"></i> Not in stock</p>\n <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star"></span><span class="fa fa-star"></span><span class="fa fa-star not_filled"></span></div>\n <p class="review green" data-rating="20">20 Reviews</p>\n <button>Add to cart</button>\n </div>\n </div>\n <div class="book_container col-sm-4">\n <img src="images/understanding OS.jpg" alt="" title="This book approaches the LINUX environment as a programming companion. No previous exposure to LINUX is necessary to utilize this text.">\n <p class="book_name"><a href="https://www.amazon.com/Understanding-Operating-Systems-Ann-McHoes/dp/1305674251" target="_blank">Understanding operating systems</br> By Ida Flynn</a></p>\n <div class="align-left">\n <p class=" price green">Rs.2700</p>\n <p class="stock not_stock " data-stock="not in stock"><i class="fa fa-times" aria-hidden="true"></i> Not in stock</p>\n <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div>\n <p class="review green" data-rating="60">60 Reviews</p>\n <button>Add to cart</button>\n </div>\n </div>\n <div class="book_container col-sm-4">\n <img src="images/Computer Systems.jpg" alt="" title="It covers computers\' data representation, explains how computer architecture and its underlying meaning changed over the years, and examines the implementations and performance enhancements of the central processing unit (CPU).">\n <p class="book_name"><a href="https://www.goodreads.com/book/show/829182.Computer_Systems" target="_blank">Computer Systems </br> By Randal E. Bryant </a></p>\n <div class="align-left">\n <p class=" price green">Rs.1700</p>\n <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p>\n <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div>\n <p class="review green" data-rating="25">25 Reviews</p>\n <button>Add to cart</button>\n </div>\n </div>\n <div class="book_container col-sm-4">\n <img src="images/linux bible.jpg" alt="" title="It includes information on cloud computing, with new guidance on containerization, Ansible automation, and Kubernetes and OpenShift. With a focus on RHEL 8">\n <p class="book_name"><a href="https://www.amazon.com/Linux-Bible-Christopher-Negus/dp/111821854X" target="_blank">Linux bible </br> Book By Christopher Negus</a></p></p>\n <div class="align-left">\n <p class=" price green">Rs.1800</p>\n <p class="stock not_stock " data-stock="not in stock"><i class="fa fa-times" aria-hidden="true"></i> Not in stock</p>\n <div><span class="fa fa-star"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div>\n <p class="review green" data-rating="21">21 Reviews</p>\n <button>Add to cart</button>\n </div>\n </div>\n <div class="book_container col-sm-4">\n <img src="images/advanced programming in unix.jpg" alt="" title="This book approaches the LINUX environment as a programming companion. No previous exposure to LINUX is necessary to utilize this text.">\n <p class="book_name"><a href="https://www.amazon.com/dp/0321637739?tag=uuid10-20" target="_blank">Advanced Programming in the UNIX Environment </br> By W. Stevans</a></p>\n <div class="align-left">\n <p class=" price green">Rs.6000</p>\n <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p>\n <div><span class="fa fa-star"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div>\n <p class="review green" data-rating="40">40 Reviews</p>\n <button>Add to cart</button>\n </div>\n </div>\n <div class="book_container col-sm-4">\n <img src="images/OS a design appraoch.png" alt="" title="It covers computers\' data representation, explains how computer architecture and its underlying meaning changed over the years, and examines the implementations and performance enhancements of the central processing unit (CPU).">\n <p class="book_name"><a href="https://www.amazon.com/s?k=Operating+Systems%3A+A+Design-oriented+Approach&i=stripbooks-intl-ship&ref=nb_sb_noss" target="_blank">Operating Systems: A Design-oriented Approach</br> By Charles Patrick Crowley</a></p>\n <div class="align-left">\n <p class=" price green">Rs.1000</p>\n <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p>\n <div><span class="fa fa-star"></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div>\n <p class="review green" data-rating="90">90 Reviews</p>\n <button>Add to cart</button>\n </div>\n </div>\n <div class="book_container col-sm-4">\n <img src="images/hands on network programming.jpg" alt="" title="It includes information on cloud computing, with new guidance on containerization, Ansible automation, and Kubernetes and OpenShift. With a focus on RHEL 8">\n <p class="book_name"><a href="https://www.amazon.com/Hands-Network-Programming-programming-optimized/dp/1789349869/ref=sr_1_1?crid=11FC0M0GAFA21&&keywords=unix+network+programming+2019&&qid=1653381349&&s=books&&sprefix=unix+network+programming+2019%2Cstripbooks-intl-ship%2C356&&sr=1-1" target="_blank">Hands-On Network Programming with C </br> By Lewis Van Winkle</p></a></p>\n <div class="align-left">\n <p class=" price green">Rs.1800</p>\n <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p>\n <div><span class="fa fa-star"></span><span class="fa fa-star "></span><span class="fa fa-star"></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span></div>\n <p class="review green" data-rating="70">70 Reviews</p>\n <button>Add to cart</button>\n </div>\n </div>\n </div>\n </div>\n </section>\n <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/js/bootstrap.bundle.min.js" integrity="sha384-pprn3073KE6tl6bjs2QrFaJGz5/SUsLqktiwsUTF55Jfv3qYSDhgCecCxMW52nD2" crossorigin="anonymous"></script>\n </body>\n</html>\n'
Requests
Library¶Requests
is an open-source Python library that makes HTTP requests more human-friendly and simple to use.
The requests
library support a fully RESTful API and uses commands to obtain resources, such as:
request.get()
to retrieve a resource;request.put()
to change the state of or update a resource, which can be an object, file or blockrequest.post()
to create that resourcerequest.delete()
to remove a resource.A REST API is an API that conforms to the design principles of the REST, or representational state transfer architectural style:
Installing Requests Library
import sys
!{sys.executable} -m pip install --upgrade pip -q
!{sys.executable} -m pip install --upgrade requests -q
import requests
requests.__version__ , requests.__path__
('2.28.1', ['/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/requests'])
Example 1: Fetching, Reading and saving the HTML code of a Web page in a Text File
print(dir(requests))
['ConnectTimeout', 'ConnectionError', 'DependencyWarning', 'FileModeWarning', 'HTTPError', 'JSONDecodeError', 'NullHandler', 'PreparedRequest', 'ReadTimeout', 'Request', 'RequestException', 'RequestsDependencyWarning', 'Response', 'Session', 'Timeout', 'TooManyRedirects', 'URLRequired', '__author__', '__author_email__', '__build__', '__builtins__', '__cached__', '__cake__', '__copyright__', '__description__', '__doc__', '__file__', '__license__', '__loader__', '__name__', '__package__', '__path__', '__spec__', '__title__', '__url__', '__version__', '_check_cryptography', '_internal_utils', 'adapters', 'api', 'auth', 'certs', 'chardet_version', 'charset_normalizer_version', 'check_compatibility', 'codes', 'compat', 'cookies', 'delete', 'exceptions', 'get', 'head', 'hooks', 'logging', 'models', 'options', 'packages', 'patch', 'post', 'put', 'request', 'session', 'sessions', 'ssl', 'status_codes', 'structures', 'urllib3', 'utils', 'warnings']
resp = requests.get("https://arifpucit.github.io/bss2")
print(type(resp))
<class 'requests.models.Response'>
print(dir(resp))
['__attrs__', '__bool__', '__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__enter__', '__eq__', '__exit__', '__format__', '__ge__', '__getattribute__', '__getstate__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__iter__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__nonzero__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__setstate__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_content', '_content_consumed', '_next', 'apparent_encoding', 'close', 'connection', 'content', 'cookies', 'elapsed', 'encoding', 'headers', 'history', 'is_permanent_redirect', 'is_redirect', 'iter_content', 'iter_lines', 'json', 'links', 'next', 'ok', 'raise_for_status', 'raw', 'reason', 'request', 'status_code', 'text', 'url']
resp.status_code
200
resp.url
'https://arifpucit.github.io/bss2/'
resp.headers
{'Connection': 'keep-alive', 'Content-Length': '2757', 'Server': 'GitHub.com', 'Content-Type': 'text/html; charset=utf-8', 'permissions-policy': 'interest-cohort=()', 'Last-Modified': 'Mon, 27 Jun 2022 12:32:49 GMT', 'Access-Control-Allow-Origin': '*', 'ETag': 'W/"62b9a371-33ad"', 'expires': 'Sun, 31 Jul 2022 05:52:04 GMT', 'Cache-Control': 'max-age=600', 'Content-Encoding': 'gzip', 'x-proxy-cache': 'MISS', 'X-GitHub-Request-Id': 'DB9E:6669:1DA18F:24947F:62E6162C', 'Accept-Ranges': 'bytes', 'Date': 'Sun, 31 Jul 2022 09:32:02 GMT', 'Via': '1.1 varnish', 'Age': '0', 'X-Served-By': 'cache-qpg1243-QPG', 'X-Cache': 'HIT', 'X-Cache-Hits': '1', 'X-Timer': 'S1659259922.301084,VS0,VE235', 'Vary': 'Accept-Encoding', 'X-Fastly-Request-ID': '849901c2a1026515f3fef3acdc9c36a848c8f5d5'}
resp.headers['Server']
'GitHub.com'
resp.headers['Last-Modified']
'Mon, 27 Jun 2022 12:32:49 GMT'
print(resp.text)
<!doctype html> <html lang="en"> <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>BSS2</title> <!-- external style sheet --> <link rel="stylesheet" href="./index.css"> <!--Bootstrap style sheet--> <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-0evHe/X+R7YkIZDRvuzKMRqM+OrBnVFBL6DOitfPri4tjfHxaWutUpFmBp4vmVor" crossorigin="anonymous"> <!--for icons of tick and cross ans star for in stock--> <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css"> </head> <body> <header class="header d-flex align-items-center justify-content-between"> <img class="image-container" src="./images//arif.jpg" alt="arif"/> <p> <span class="large_text">Books Scraping Site</span></p> <img class="image-container" src="./images/pucit.jpg" alt="pucit"/> </header> <section> <div class="main-container d-flex align-items-start justify-content-between"> <div class="navbar"> <ul class="nav-links"> <div class="link text-center" id="book_title">Books Titles</div> <li class="link book_type"><a href="index.html">Operating System</a></li> <li class="link book_type"><a href="SP.html">System Programming</a></li> <li class="link book_type"><a href="CA.html">Computer Architecture</a></li> </ul> </div> <div class="items"> <div id="book_page_titile">Operating Systems</div> <div class="row"> <div class="book_container col-sm-4"> <img src="images/OS concepts.jpg" alt="" title="The Linux Programming Interface (TLPI) is the definitive guide to the Linux and UNIX programming interface—the interface employed by nearly every application that runs on a Linux or UNIX system."> <p class="book_name"><a href="https://www.amazon.com/Operating-System-Concepts-Abridged-Companion/dp/1119456339" target="_blank">Operating System Concepts </br>By Avi Silberschatz</a></p> <div class="align-left"> <p class=" price green">Rs.2000</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="20">20 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/Unix.jpg" alt="" title="It covers many topics not covered in older, more traditional textbook approaches, such as Python, UNIX System Programming from basics to socket-based network programming using the client-server paradigm, the Zettabyte File System (ZFS), and the highly developed X Windows-based KDE and Gnome GUI desktop environments"> <p class="book_name"><a href="https://www.google.com/search?q=Unix+the+textbook+by+mansoor&rlz=1C1CHBD_enPK987PK987&oq=unix+the+textbook+by+mansoor&aqs=chrome.0.69i59j69i57j69i59j69i60l5.4419j0j7&sourceid=chrome&ie=UTF-8" target="_blank">UNIX The Textbook</br> By Syed Mansoor Sarwar</a></p> <div class="align-left"> <p class=" price green">Rs.5000</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span></div> <p class="review green" data-rating="100">100 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/taxonomy.jpg" alt="" title="Intrusion Detection using System Call Sequence"> <p class="book_name"><a href="https://www.amazon.in/Taxonomy-Ids-Arif-Butt/dp/3639294092" target="_blank">Taxonomy of IDS </br>By Arif Butt</a></p> <div class="align-left"> <p class=" price green">Rs.6900</p> <p class="stock not_stock " data-stock="not in stock"><i class="fa fa-times" aria-hidden="true"></i> Not in stock</p> <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star"></span><span class="fa fa-star"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="20">20 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/understanding OS.jpg" alt="" title="This book approaches the LINUX environment as a programming companion. No previous exposure to LINUX is necessary to utilize this text."> <p class="book_name"><a href="https://www.amazon.com/Understanding-Operating-Systems-Ann-McHoes/dp/1305674251" target="_blank">Understanding operating systems</br> By Ida Flynn</a></p> <div class="align-left"> <p class=" price green">Rs.2700</p> <p class="stock not_stock " data-stock="not in stock"><i class="fa fa-times" aria-hidden="true"></i> Not in stock</p> <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="60">60 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/Computer Systems.jpg" alt="" title="It covers computers' data representation, explains how computer architecture and its underlying meaning changed over the years, and examines the implementations and performance enhancements of the central processing unit (CPU)."> <p class="book_name"><a href="https://www.goodreads.com/book/show/829182.Computer_Systems" target="_blank">Computer Systems </br> By Randal E. Bryant </a></p> <div class="align-left"> <p class=" price green">Rs.1700</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="25">25 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/linux bible.jpg" alt="" title="It includes information on cloud computing, with new guidance on containerization, Ansible automation, and Kubernetes and OpenShift. With a focus on RHEL 8"> <p class="book_name"><a href="https://www.amazon.com/Linux-Bible-Christopher-Negus/dp/111821854X" target="_blank">Linux bible </br> Book By Christopher Negus</a></p></p> <div class="align-left"> <p class=" price green">Rs.1800</p> <p class="stock not_stock " data-stock="not in stock"><i class="fa fa-times" aria-hidden="true"></i> Not in stock</p> <div><span class="fa fa-star"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="21">21 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/advanced programming in unix.jpg" alt="" title="This book approaches the LINUX environment as a programming companion. No previous exposure to LINUX is necessary to utilize this text."> <p class="book_name"><a href="https://www.amazon.com/dp/0321637739?tag=uuid10-20" target="_blank">Advanced Programming in the UNIX Environment </br> By W. Stevans</a></p> <div class="align-left"> <p class=" price green">Rs.6000</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="40">40 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/OS a design appraoch.png" alt="" title="It covers computers' data representation, explains how computer architecture and its underlying meaning changed over the years, and examines the implementations and performance enhancements of the central processing unit (CPU)."> <p class="book_name"><a href="https://www.amazon.com/s?k=Operating+Systems%3A+A+Design-oriented+Approach&i=stripbooks-intl-ship&ref=nb_sb_noss" target="_blank">Operating Systems: A Design-oriented Approach</br> By Charles Patrick Crowley</a></p> <div class="align-left"> <p class=" price green">Rs.1000</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star"></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="90">90 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/hands on network programming.jpg" alt="" title="It includes information on cloud computing, with new guidance on containerization, Ansible automation, and Kubernetes and OpenShift. With a focus on RHEL 8"> <p class="book_name"><a href="https://www.amazon.com/Hands-Network-Programming-programming-optimized/dp/1789349869/ref=sr_1_1?crid=11FC0M0GAFA21&&keywords=unix+network+programming+2019&&qid=1653381349&&s=books&&sprefix=unix+network+programming+2019%2Cstripbooks-intl-ship%2C356&&sr=1-1" target="_blank">Hands-On Network Programming with C </br> By Lewis Van Winkle</p></a></p> <div class="align-left"> <p class=" price green">Rs.1800</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star"></span><span class="fa fa-star "></span><span class="fa fa-star"></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="70">70 Reviews</p> <button>Add to cart</button> </div> </div> </div> </div> </section> <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/js/bootstrap.bundle.min.js" integrity="sha384-pprn3073KE6tl6bjs2QrFaJGz5/SUsLqktiwsUTF55Jfv3qYSDhgCecCxMW52nD2" crossorigin="anonymous"></script> </body> </html>
with open("bss2-index.html", 'wt') as fd:
fd.write(resp.text)
!cat bss2-index.html
<!doctype html> <html lang="en"> <head> <meta charset="utf-8"> <meta name="viewport" content="width=device-width, initial-scale=1"> <title>BSS2</title> <!-- external style sheet --> <link rel="stylesheet" href="./index.css"> <!--Bootstrap style sheet--> <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-0evHe/X+R7YkIZDRvuzKMRqM+OrBnVFBL6DOitfPri4tjfHxaWutUpFmBp4vmVor" crossorigin="anonymous"> <!--for icons of tick and cross ans star for in stock--> <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css"> </head> <body> <header class="header d-flex align-items-center justify-content-between"> <img class="image-container" src="./images//arif.jpg" alt="arif"/> <p> <span class="large_text">Books Scraping Site</span></p> <img class="image-container" src="./images/pucit.jpg" alt="pucit"/> </header> <section> <div class="main-container d-flex align-items-start justify-content-between"> <div class="navbar"> <ul class="nav-links"> <div class="link text-center" id="book_title">Books Titles</div> <li class="link book_type"><a href="index.html">Operating System</a></li> <li class="link book_type"><a href="SP.html">System Programming</a></li> <li class="link book_type"><a href="CA.html">Computer Architecture</a></li> </ul> </div> <div class="items"> <div id="book_page_titile">Operating Systems</div> <div class="row"> <div class="book_container col-sm-4"> <img src="images/OS concepts.jpg" alt="" title="The Linux Programming Interface (TLPI) is the definitive guide to the Linux and UNIX programming interface—the interface employed by nearly every application that runs on a Linux or UNIX system."> <p class="book_name"><a href="https://www.amazon.com/Operating-System-Concepts-Abridged-Companion/dp/1119456339" target="_blank">Operating System Concepts </br>By Avi Silberschatz</a></p> <div class="align-left"> <p class=" price green">Rs.2000</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="20">20 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/Unix.jpg" alt="" title="It covers many topics not covered in older, more traditional textbook approaches, such as Python, UNIX System Programming from basics to socket-based network programming using the client-server paradigm, the Zettabyte File System (ZFS), and the highly developed X Windows-based KDE and Gnome GUI desktop environments"> <p class="book_name"><a href="https://www.google.com/search?q=Unix+the+textbook+by+mansoor&rlz=1C1CHBD_enPK987PK987&oq=unix+the+textbook+by+mansoor&aqs=chrome.0.69i59j69i57j69i59j69i60l5.4419j0j7&sourceid=chrome&ie=UTF-8" target="_blank">UNIX The Textbook</br> By Syed Mansoor Sarwar</a></p> <div class="align-left"> <p class=" price green">Rs.5000</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star "></span></div> <p class="review green" data-rating="100">100 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/taxonomy.jpg" alt="" title="Intrusion Detection using System Call Sequence"> <p class="book_name"><a href="https://www.amazon.in/Taxonomy-Ids-Arif-Butt/dp/3639294092" target="_blank">Taxonomy of IDS </br>By Arif Butt</a></p> <div class="align-left"> <p class=" price green">Rs.6900</p> <p class="stock not_stock " data-stock="not in stock"><i class="fa fa-times" aria-hidden="true"></i> Not in stock</p> <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star"></span><span class="fa fa-star"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="20">20 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/understanding OS.jpg" alt="" title="This book approaches the LINUX environment as a programming companion. No previous exposure to LINUX is necessary to utilize this text."> <p class="book_name"><a href="https://www.amazon.com/Understanding-Operating-Systems-Ann-McHoes/dp/1305674251" target="_blank">Understanding operating systems</br> By Ida Flynn</a></p> <div class="align-left"> <p class=" price green">Rs.2700</p> <p class="stock not_stock " data-stock="not in stock"><i class="fa fa-times" aria-hidden="true"></i> Not in stock</p> <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="60">60 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/Computer Systems.jpg" alt="" title="It covers computers' data representation, explains how computer architecture and its underlying meaning changed over the years, and examines the implementations and performance enhancements of the central processing unit (CPU)."> <p class="book_name"><a href="https://www.goodreads.com/book/show/829182.Computer_Systems" target="_blank">Computer Systems </br> By Randal E. Bryant </a></p> <div class="align-left"> <p class=" price green">Rs.1700</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="25">25 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/linux bible.jpg" alt="" title="It includes information on cloud computing, with new guidance on containerization, Ansible automation, and Kubernetes and OpenShift. With a focus on RHEL 8"> <p class="book_name"><a href="https://www.amazon.com/Linux-Bible-Christopher-Negus/dp/111821854X" target="_blank">Linux bible </br> Book By Christopher Negus</a></p></p> <div class="align-left"> <p class=" price green">Rs.1800</p> <p class="stock not_stock " data-stock="not in stock"><i class="fa fa-times" aria-hidden="true"></i> Not in stock</p> <div><span class="fa fa-star"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="21">21 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/advanced programming in unix.jpg" alt="" title="This book approaches the LINUX environment as a programming companion. No previous exposure to LINUX is necessary to utilize this text."> <p class="book_name"><a href="https://www.amazon.com/dp/0321637739?tag=uuid10-20" target="_blank">Advanced Programming in the UNIX Environment </br> By W. Stevans</a></p> <div class="align-left"> <p class=" price green">Rs.6000</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="40">40 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/OS a design appraoch.png" alt="" title="It covers computers' data representation, explains how computer architecture and its underlying meaning changed over the years, and examines the implementations and performance enhancements of the central processing unit (CPU)."> <p class="book_name"><a href="https://www.amazon.com/s?k=Operating+Systems%3A+A+Design-oriented+Approach&i=stripbooks-intl-ship&ref=nb_sb_noss" target="_blank">Operating Systems: A Design-oriented Approach</br> By Charles Patrick Crowley</a></p> <div class="align-left"> <p class=" price green">Rs.1000</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star"></span><span class="fa fa-star "></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="90">90 Reviews</p> <button>Add to cart</button> </div> </div> <div class="book_container col-sm-4"> <img src="images/hands on network programming.jpg" alt="" title="It includes information on cloud computing, with new guidance on containerization, Ansible automation, and Kubernetes and OpenShift. With a focus on RHEL 8"> <p class="book_name"><a href="https://www.amazon.com/Hands-Network-Programming-programming-optimized/dp/1789349869/ref=sr_1_1?crid=11FC0M0GAFA21&&keywords=unix+network+programming+2019&&qid=1653381349&&s=books&&sprefix=unix+network+programming+2019%2Cstripbooks-intl-ship%2C356&&sr=1-1" target="_blank">Hands-On Network Programming with C </br> By Lewis Van Winkle</p></a></p> <div class="align-left"> <p class=" price green">Rs.1800</p> <p class="stock in_stock " data-stock="in stock"><i class="fa fa-check" aria-hidden="true"></i> In stock</p> <div><span class="fa fa-star"></span><span class="fa fa-star "></span><span class="fa fa-star"></span><span class="fa fa-star "></span><span class="fa fa-star not_filled"></span></div> <p class="review green" data-rating="70">70 Reviews</p> <button>Add to cart</button> </div> </div> </div> </div> </section> <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.0-beta1/dist/js/bootstrap.bundle.min.js" integrity="sha384-pprn3073KE6tl6bjs2QrFaJGz5/SUsLqktiwsUTF55Jfv3qYSDhgCecCxMW52nD2" crossorigin="anonymous"></script> </body> </html>
Example 2: Fetching, Reading and saving an Image from the Web Page in a Binary File
url = "https://arifpucit.github.io/bss2/" + "./images/arif.jpg"
resp = requests.get(url)
resp.status_code
200
print(resp.text)
���� JFIF �� � !.%+&8&+/1555$;@;4?.4514!%$14441444444144441144444444444444144444444444444141�� � �"