from datetime import datetime
start_time = datetime.now()
print("Last run: ", start_time)
Last run: 2023-08-07 13:43:30.655973
import urllib3, json
import pandas as pd
http = urllib3.PoolManager()
url= "https://portrattarkiv.se/endpoints/search.php"
import sys
from SPARQLWrapper import SPARQLWrapper, JSON
endpoint_url = "https://query.wikidata.org/sparql"
# IFKDB personer födda innan 1920 utan bild - SPA innehåller mest gamla personer
queryIFKDB = """SELECT (CONCAT (?itemLabel," ",str(year(?birthDate))) AS ?search) ?item ?itemLabel
(str(year(?birthDate)) AS ?year)
WHERE {
?item wdt:P11905 ?ifkdb.
minus{?item wdt:P19 ?img} # saknar bild
?item wdt:P569 ?birthDate.
FILTER(xsd:integer(year(?birthDate)) < 1920)
SERVICE wikibase:label { bd:serviceParam wikibase:language "sv", "en". }
} order by ?itemLabel """
def get_sparql_dataframe(endpoint_url, query):
"""
Helper function to convert SPARQL results into a Pandas data frame.
"""
user_agent = "salgo60/%s.%s" % (sys.version_info[0], sys.version_info[1])
sparql = SPARQLWrapper(endpoint_url, agent=user_agent)
sparql.setQuery(query)
sparql.setReturnFormat(JSON)
result = sparql.query()
processed_results = json.load(result.response)
cols = processed_results['head']['vars']
out = []
for row in processed_results['results']['bindings']:
item = []
for c in cols:
item.append(row.get(c, {}).get('value'))
out.append(item)
return pd.DataFrame(out, columns=cols)
#SPAdf = get_sparql_dataframe(endpoint_url, query)
#SPAdf = get_sparql_dataframe(endpoint_url, queryNoPicNorra1890)
SPAdf = get_sparql_dataframe(endpoint_url, queryIFKDB)
SPAdf.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 11 entries, 0 to 10 Data columns (total 4 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 search 11 non-null object 1 item 11 non-null object 2 itemLabel 11 non-null object 3 year 11 non-null object dtypes: object(4) memory usage: 480.0+ bytes
SPAdf.shape
(11, 4)
SPAdf
| search | item | itemLabel | year | |
|---|---|---|---|---|
| 0 | Douglas Krook 1897 | http://www.wikidata.org/entity/Q1252363 | Douglas Krook | 1897 |
| 1 | Elis Berndtsson 1902 | http://www.wikidata.org/entity/Q121160724 | Elis Berndtsson | 1902 |
| 2 | Erik Eiserman 1892 | http://www.wikidata.org/entity/Q121160732 | Erik Eiserman | 1892 |
| 3 | Gustaf Hermansson 1900 | http://www.wikidata.org/entity/Q121160746 | Gustaf Hermansson | 1900 |
| 4 | Gösta Bohman 1911 | http://www.wikidata.org/entity/Q121160725 | Gösta Bohman | 1911 |
| 5 | Göte Andersson 1919 | http://www.wikidata.org/entity/Q121160713 | Göte Andersson | 1919 |
| 6 | Helge Andersson 1899 | http://www.wikidata.org/entity/Q121160714 | Helge Andersson | 1899 |
| 7 | John Andersson 1914 | http://www.wikidata.org/entity/Q121160715 | John Andersson | 1914 |
| 8 | John Säwström 1886 | http://www.wikidata.org/entity/Q121052611 | John Säwström | 1886 |
| 9 | Olof Hjärn 1903 | http://www.wikidata.org/entity/Q121160747 | Olof Hjärn | 1903 |
| 10 | Vallentin Hultén 1911 | http://www.wikidata.org/entity/Q121160750 | Vallentin Hultén | 1911 |
SPAdetail = "https://portrattarkiv.se/details/"
for index, row in SPAdf.iterrows():
print("\n",row["search"],row["item"])
encoded_body = json.dumps({
"limit": "5",
"from": "0",
"birthyear":row["year"],
"all":row["search"]
})
r = http.request('POST', url,
headers={'Content-Type': 'application/json'},
body=encoded_body)
if r.status != 200:
print(r.status)
continue
data = json.loads(r.data.decode('utf-8'),)
urls = []
for h in data["hits"]["hits"]:
id = h["_id"]
#print(h)
source = h["_source"]
try:
# urlPicture = urlbasePic + id
# urls.append(urlPicture)
score = h["_score"]
FirstName = source["FirstName"]
LastName = source["LastName"]
BirthYear = source["BirthYear"]
print("\t\t",score,FirstName, " ", LastName, " - ", BirthYear,SPAdetail+id, "\t", )
except:
print("Error")
Douglas Krook 1897 http://www.wikidata.org/entity/Q1252363 20.313782 Karl Arnold Douglas Larsson - 1897 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAACiN4A 19.261509 Albert (Abbe) Ragnar Krook - 1897 https://portrattarkiv.se/details/0piq-fX0h5AAAAAAAABkjw 19.255459 Douglas Olsson - 1897 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAACet2g 18.22902 Douglas Bondeman - 1897 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA2JNg 18.155508 Douglas Wallhäll - 1897 https://portrattarkiv.se/details/ciLMGSqrYjAAAAAAAABicA Elis Berndtsson 1902 http://www.wikidata.org/entity/Q121160724 20.571924 Erland Aloys Ingvar Berndtsson - 1902 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAACiLnw 19.767786 Ellen Nilsson-Berndtsson - 1902 https://portrattarkiv.se/details/wbbNj8LLWMAAAAAAAAAoEA 18.01412 Carl Berndtsson - 1902 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABVsXw 17.49529 Elis Nestor - 1902 https://portrattarkiv.se/details/wbbNj8LLWMAAAAAAAABfvA 17.31288 Elis Larsson - 1902 https://portrattarkiv.se/details/rkIpfGqHXrAAAAAAAAAWjA Erik Eiserman 1892 http://www.wikidata.org/entity/Q121160732 12.96698 Erik Reichenberg - 1892 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABMClQ 12.910355 Erik Akmar - 1892 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA2VFw 12.852482 Erik Weis - 1892 https://portrattarkiv.se/details/ciLMGSqrYjAAAAAAAAAiIA 12.829615 Erik Engberg - 1892 https://portrattarkiv.se/details/TR3yO_GV8eAAAAAAAAA13A 12.806978 Erik Renström - 1892 https://portrattarkiv.se/details/9Y5iReTPHdAAAAAAAABCxQ Gustaf Hermansson 1900 http://www.wikidata.org/entity/Q121160746 19.042025 Kurt Hermansson - 1900 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAACZjsA 18.20328 Tage Hermansson - 1900 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAABQXw 17.341541 Gustav Hermansson - 1900 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAACWlw 17.121572 Helge Hermansson - 1900 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAABQpw 17.114563 Anders Hermansson - 1900 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAABRaA Gösta Bohman 1911 http://www.wikidata.org/entity/Q121160725 23.806688 Gösta Bohman - 1911 https://portrattarkiv.se/details/4e1scVvZ-EAAAAAAAAAQ5A 14.541103 Gösta Johansson - 1911 https://portrattarkiv.se/details/Di4a42b6X_AAAAAAAACSmA 14.473673 Gösta Morberg - 1911 https://portrattarkiv.se/details/I_2wtd5VefAAAAAAAABDng 14.352381 Gösta Fall - 1911 https://portrattarkiv.se/details/IQojCnw0WmAAAAAAAAAWwQ 14.129609 Gösta Morberg - 1911 https://portrattarkiv.se/details/I_2wtd5VefAAAAAAAABDnw Göte Andersson 1919 http://www.wikidata.org/entity/Q121160713 21.547554 Göte Andersson - 1919 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABVyQQ 20.95187 Göte Andersson - 1919 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABVTJA 20.342884 Göte Andersson - 1919 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA5NIA 18.98707 Göte Andersson - 1919 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA5NIQ 18.29131 Göte Andersson - 1919 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABGTIw Helge Andersson 1899 http://www.wikidata.org/entity/Q121160714 18.763737 Karl Andersson - 1899 https://portrattarkiv.se/details/lzSYNK3ifoMAAAAAAAY-Hg 18.60417 Helge Andersson - 1899 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA5YWg 18.57062 Helge Andersson - 1899 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA5YXA 18.36661 Helge Andersson - 1899 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA5YWw 17.768845 Helge Karlsson - 1899 https://portrattarkiv.se/details/seuNpYu6DKAAAAAAAABN7A John Andersson 1914 http://www.wikidata.org/entity/Q121160715 16.870964 John Andersson - 1914 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA5Oyg 16.133652 John Andersson - 1914 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA5Oyw 15.980896 John Tjäder - 1914 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABzoOQ 15.449126 John Nilsson - 1914 https://portrattarkiv.se/details/wbbNj8LLWMAAAAAAAAB_nA 14.804291 Maria Andersson - 1914 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAAA4-9w John Säwström 1886 http://www.wikidata.org/entity/Q121052611 14.925089 John Holmberg - 1886 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABe3DA 14.837259 John Nilsson - 1886 https://portrattarkiv.se/details/wbbNj8LLWMAAAAAAAAAV9g 14.750305 John Lundberg - 1886 https://portrattarkiv.se/details/rkIpfGqHXrAAAAAAAACMSA 14.71521 John Nordlander - 1886 https://portrattarkiv.se/details/wbbNj8LLWMAAAAAAAAA91A 14.693815 John Johansson - 1886 https://portrattarkiv.se/details/Di4a42b6X_AAAAAAAAAnxQ Olof Hjärn 1903 http://www.wikidata.org/entity/Q121160747 13.323799 Olof Lignell - 1903 https://portrattarkiv.se/details/rkIpfGqHXrAAAAAAAAA2AQ 13.184191 Olof Olsson - 1903 https://portrattarkiv.se/details/vfWYdYT6EoAAAAAAAAAZ9w 12.992161 Carl-Olof Lindblom - 1903 https://portrattarkiv.se/details/rkIpfGqHXrAAAAAAAABH8Q 12.977806 Olof Widestrand - 1903 https://portrattarkiv.se/details/ciLMGSqrYjAAAAAAAABAtQ 12.959324 Olof Ekström - 1903 https://portrattarkiv.se/details/TR3yO_GV8eAAAAAAAABffQ Vallentin Hultén 1911 http://www.wikidata.org/entity/Q121160750 20.69121 Vivi-Anne Hultén - 1911 https://portrattarkiv.se/details/j6S8rJvx9lAAAAAAAAC3rQ 20.69121 Vivi-Anne Hultén - 1911 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABO32A 20.670612 Gösta Hultén - 1911 https://portrattarkiv.se/details/seuNpYu6DKAAAAAAAAAK2A 20.37493 Vivi-Anne Hultén - 1911 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAACY1Aw 20.285803 Gösta Hultén - 1911 https://portrattarkiv.se/details/sj9PGLAlnmUAAAAAABVCiA