from datetime import datetime
start_time = datetime.now()
print("Last run: ", start_time)
Last run: 2023-01-03 10:43:12.919072
# pip install sparqlwrapper
# https://rdflib.github.io/sparqlwrapper/
import pandas as pd
import sys,json
from SPARQLWrapper import SPARQLWrapper, JSON
endpoint_url = "https://query.wikidata.org/sparql"
#https://w.wiki/4J4B
queryLittNoPic = """SELECT ?item ?itemLabel (year(?born) as ?year) ?statements ?Litteraturbankenid
(CONCAT(?itemLabel," ",str(year(?born))) AS ?search) WHERE {
?item wdt:P5101 ?Litteraturbankenid ;
wikibase:statements ?statements .
# FILTER (?statements <100)
minus {?item wdt:P18 ?img}
OPTIONAL{?item wdt:P569 ?born}
FILTER (year(?born) < 1900)
FILTER (year(?born) > 1800)
SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
} order by desc(?statements)
"""
def get_results(endpoint_url, query):
user_agent = "WDQS-example Python/%s.%s" % (sys.version_info[0], sys.version_info[1])
# TODO adjust user agent; see https://w.wiki/CX6
sparql = SPARQLWrapper(endpoint_url, agent=user_agent)
sparql.setQuery(query)
sparql.setReturnFormat(JSON)
return sparql.query().convert()
results = get_results(endpoint_url, queryLittNoPic)
def get_sparql_dataframe(endpoint_url, query):
"""
Helper function to convert SPARQL results into a Pandas data frame.
"""
user_agent = "salgo60/%s.%s" % (sys.version_info[0], sys.version_info[1])
sparql = SPARQLWrapper(endpoint_url, agent=user_agent)
sparql.setQuery(query)
sparql.setReturnFormat(JSON)
result = sparql.query()
processed_results = json.load(result.response)
cols = processed_results['head']['vars']
out = []
for row in processed_results['results']['bindings']:
item = []
for c in cols:
item.append(row.get(c, {}).get('value'))
out.append(item)
return pd.DataFrame(out, columns=cols)
WDLittNoPic = get_sparql_dataframe(endpoint_url, queryLittNoPic)
WDLittNoPic.shape
(395, 6)
WDLittNoPic
item | itemLabel | year | statements | Litteraturbankenid | search | |
---|---|---|---|---|---|---|
0 | http://www.wikidata.org/entity/Q1787190 | Mathilde Prager | 1844 | 66 | HolmErich | Mathilde Prager 1844 |
1 | http://www.wikidata.org/entity/Q20652547 | Franz Hoffmann | 1814 | 55 | HoffmannF | Franz Hoffmann 1814 |
2 | http://www.wikidata.org/entity/Q1744240 | Laura Kieler | 1849 | 48 | KielerL | Laura Kieler 1849 |
3 | http://www.wikidata.org/entity/Q3121041 | Gustaf Nordenskiöld | 1868 | 48 | NordenskiöldG | Gustaf Nordenskiöld 1868 |
4 | http://www.wikidata.org/entity/Q389396 | Johan Andersson | 1820 | 46 | AnderssonJohan | Johan Andersson 1820 |
... | ... | ... | ... | ... | ... | ... |
390 | http://www.wikidata.org/entity/Q52577083 | Gunnar Wetterberg | 1863 | 5 | WetterbergG | Gunnar Wetterberg 1863 |
391 | http://www.wikidata.org/entity/Q52690406 | J.E. Ahlstrand | 1814 | 5 | AhlstrandJE | J.E. Ahlstrand 1814 |
392 | http://www.wikidata.org/entity/Q114115033 | Josef Jonsson | 1850 | 5 | JonssonJ | Josef Jonsson 1850 |
393 | http://www.wikidata.org/entity/Q115958630 | Q115958630 | 1867 | 4 | SalomonssonS | Q115958630 1867 |
394 | http://www.wikidata.org/entity/Q100322150 | M. L. Sunder | 1840 | 3 | SunderML | M. L. Sunder 1840 |
395 rows × 6 columns
import urllib3, json
http = urllib3.PoolManager()
SPAdetail = "https://portrattarkiv.se/details/"
url= "https://portrattarkiv.se/endpoints/search.php"
urlbasePic = "https://portrattarkiv.se/endpoints/file.php?id="
for index, row in WDLittNoPic.iterrows():
#print("\n",row["search"],row["item"])
encoded_body = json.dumps({
"limit": "5",
"from": "0",
"birthyear":row["year"],
"all":row["search"]
})
firstHeader = True # track header
r = http.request('POST', url,
headers={'Content-Type': 'application/json'},
body=encoded_body)
if r.status != 200:
print(r.status)
continue
data = json.loads(r.data.decode('utf-8'),)
urls = []
for h in data["hits"]["hits"]:
id = h["_id"]
#print(h)
source = h["_source"]
try:
urlPicture = urlbasePic + id
# urls.append(urlPicture)
score = h["_score"]
FirstName = source["FirstName"]
LastName = source["LastName"]
BirthYear = source["BirthYear"]
if score > 30:
if firstHeader:
print("\n",row["search"],row["item"])
firstHeader = False # track header
print("\t\t",score,FirstName, " ", LastName, " - ", BirthYear,SPAdetail+id, "\t", )
except:
print("Error")
August Zacharias Collin 1833 http://www.wikidata.org/entity/Q5610869 34.43135 August Collin - 1833 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAB2-3g Carolina von Platen 1816 http://www.wikidata.org/entity/Q16616025 33.689552 Achates von Platen - 1816 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABL9eQ August Theodor Låstbom 1815 http://www.wikidata.org/entity/Q4820829 37.157772 August Låstbom - 1815 https://portrattarkiv.se/details/WQknIBFpuuAAAAAAAAN75w Carl Magnus Stenbock 1874 http://www.wikidata.org/entity/Q52508735 33.440323 Vilfrid Stenbock - 1874 https://portrattarkiv.se/details/4e1scVvZ-EAAAAAAAAAryQ Johan G. R. Banér 1862 http://www.wikidata.org/entity/Q94538588 36.18424 Johan Banér-Runeskiöld - 1862 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABfoqQ Per Adolf Hagström 1823 http://www.wikidata.org/entity/Q108817417 30.209373 Per Hagström - 1823 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAACvdg Gotthard Virdestam 1893 http://www.wikidata.org/entity/Q6225933 37.83749 Gotthard Virdestam - 1893 https://portrattarkiv.se/details/88fEYD5OMjAAAAAAAAAFSQ Henning Wendell 1838 http://www.wikidata.org/entity/Q6235602 37.65098 Henning Wendell - 1838 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABUrhg 35.662746 Johan Wendell - 1838 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABMTRg 34.98269 Henning Wendell - 1838 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABlZvQ 32.30876 Henning Wendell - 1838 https://portrattarkiv.se/details/ciLMGSqrYjAAAAAAAAAd3w Viktor Moll 1852 http://www.wikidata.org/entity/Q108329291 32.88014 Viktor Moll - 1852 https://portrattarkiv.se/details/I_2wtd5VefAAAAAAAAAraQ 31.587963 Viktor Moll - 1852 https://portrattarkiv.se/details/I_2wtd5VefAAAAAAAAArag Johan Adolf Ziwertz 1837 http://www.wikidata.org/entity/Q113581945 34.568344 Adolf Ziwetz - 1837 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABUi1A Anders Herman Hallin 1831 http://www.wikidata.org/entity/Q108916554 36.6414 Anders Hallin - 1831 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAAATuw 31.848352 Herman Hallin - 1831 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAAATUQ Olof Fredrik Vahlin 1805 http://www.wikidata.org/entity/Q19721281 34.50354 Olof Vahlin - 1805 https://portrattarkiv.se/details/WQknIBFpuuAAAAAAAAN1JQ Hans Jakob Lundborg 1825 http://www.wikidata.org/entity/Q5966187 41.408936 Hans Jakob Lundborg - 1825 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAB4wDQ 32.482754 Hans Lundborg - 1825 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAB4vSw 32.391094 Hans Lundborg - 1825 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABHRDw 32.148323 Hans Lundborg - 1825 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAB4vRA 31.983076 Hans Lundborg - 1825 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAB4vUQ Jan Petter Rydelius 1859 http://www.wikidata.org/entity/Q108592097 30.31207 Petter Rydelius - 1859 https://portrattarkiv.se/details/9Y5iReTPHdAAAAAAAAAuvA Karl Fredrik Forsman 1848 http://www.wikidata.org/entity/Q15838424 32.78531 Karl Forsman - 1848 https://portrattarkiv.se/details/IQojCnw0WmAAAAAAAABPDQ 31.90858 Karl Forsman - 1848 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABTEgw 30.980991 Karl Forsman - 1848 https://portrattarkiv.se/details/IQojCnw0WmAAAAAAAAAhmA Reinhold Götberg 1835 http://www.wikidata.org/entity/Q115957550 32.454453 Reinhold Götberg - 1835 https://portrattarkiv.se/details/1ha4xkDj-uAAAAAAAAARjA 32.401527 Reinhold Götberg - 1835 https://portrattarkiv.se/details/1ha4xkDj-uAAAAAAAAARjQ Linnar Linnarsson 1884 http://www.wikidata.org/entity/Q47456556 43.77349 Linnar Linnarsson - 1884 https://portrattarkiv.se/details/rkIpfGqHXrAAAAAAAADZLA 41.61906 Linnar Linnarsson - 1884 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABKycQ 41.593452 Linnar Linnarsson - 1884 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABgxFg 41.425484 Linnar Linnarsson - 1884 https://portrattarkiv.se/details/rkIpfGqHXrAAAAAAAADiqA 35.36035 Linnar Linnarsson - 1884 https://portrattarkiv.se/details/rkIpfGqHXrAAAAAAAABsyQ Mortimer Hærén 1835 http://www.wikidata.org/entity/Q108869242 42.818356 Mortimer Hærén - 1835 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAACClQ 41.783653 Mortimer Hærén - 1835 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAACClA Theodor Giertz 1833 http://www.wikidata.org/entity/Q108587959 30.947475 Theodor Giertz - 1833 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABye3g Erik Högström 1820 http://www.wikidata.org/entity/Q108869019 31.475262 Erik Högström - 1820 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAACMUw Inga Schenfelt 1826 http://www.wikidata.org/entity/Q108916337 37.0202 Inga Schenfelt - 1826 https://portrattarkiv.se/details/YB0QHyfj0hAAAAAAAABgLA Rudolf Cederström 1816 http://www.wikidata.org/entity/Q111689071 31.606281 rudolf cederström - 1816 https://portrattarkiv.se/details/AUBJHcnv_WAAAAAAAAA3ig 30.414429 Claes Cederström - 1816 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABLxsA Zacharias Ludvig Barkstedt 1810 http://www.wikidata.org/entity/Q115956221 46.286995 Zacharias Barkstedt - 1810 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA4ZTQ Robert Litzén 1849 http://www.wikidata.org/entity/Q111688807 34.40908 Robert Litzén - 1849 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABUjzg 33.688652 Robert Litzén - 1849 https://portrattarkiv.se/details/rkIpfGqHXrAAAAAAAABuGg 33.688652 Robert Litzén - 1849 https://portrattarkiv.se/details/rkIpfGqHXrAAAAAAAABuGQ Alfred Elmbladh 1859 http://www.wikidata.org/entity/Q113585833 35.638065 Alfred Elmbladh - 1859 https://portrattarkiv.se/details/TR3yO_GV8eAAAAAAAABK8A 34.700752 Alfred Elmbladh - 1859 https://portrattarkiv.se/details/TR3yO_GV8eAAAAAAAABK7w 32.06327 Alfred Elmbladh - 1859 https://portrattarkiv.se/details/TR3yO_GV8eAAAAAAAABK8Q 31.728642 Malkus Elmbladh - 1859 https://portrattarkiv.se/details/TR3yO_GV8eAAAAAAAABK7g Ida Lindgren 1829 http://www.wikidata.org/entity/Q115956193 30.954487 Ida Lindgren-Nibelius - 1829 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABhDvA 30.765974 Ida Lindgren-Nibelius - 1829 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABhIXw Arvid Ragnar Isberg 1850 http://www.wikidata.org/entity/Q115956202 37.068226 Arvid Isberg - 1850 https://portrattarkiv.se/details/seuNpYu6DKAAAAAAAABm3w 35.3652 Arvid Isberg - 1850 https://portrattarkiv.se/details/nGoaRo7IPMAAAAAAAAAGdg 35.215706 Arvid Isberg - 1850 https://portrattarkiv.se/details/nGoaRo7IPMAAAAAAAAAGdw 32.620125 Arvid Isberg - 1850 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAACZ4Ug 31.54609 Ragnar Isberg - 1850 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAACZ3fw Carl Fredrik Hessling 1812 http://www.wikidata.org/entity/Q110212243 40.184845 Carl Hessling - 1812 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABOlSg Sigrid Brithelli 1855 http://www.wikidata.org/entity/Q110214514 31.52762 Sigrid Brithelli - 1855 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA2SzA
end = datetime.now()
print("Ended: ", end)
print('Time elapsed (hh:mm:ss.ms) {}'.format(datetime.now() - start_time))
Ended: 2023-01-03 08:29:44.587496 Time elapsed (hh:mm:ss.ms) 0:00:32.580947