import pandas as pd
import requests
from bs4 import BeautifulSoup
First we'll load the page.
# Note the 'id_' in the url to get the original page without the IA navigation.
response = requests.get(
"https://web.archive.org/web/20190716210347id_/http://www.naa.gov.au/collection/fact-sheets/by-number/index.aspx"
)
soup = BeautifulSoup(response.content)
Then we'll extract the rows from the index table.
fs_list = soup.find("table", title="Numerical list of fact sheets").find_all("tr")[1:]
Let's loop through all the rows in the fact sheet index, extracting the fact sheet number, title and url. Then we'll try loading the url. We'll save all the details and the HTTP status code for further exploration.
fact_sheets = []
for row in fs_list:
num = row.td.text
fs = row.find("a")
title = fs.text
url = f'http://naa.gov.au{fs["href"]}'
response = requests.get(url)
status = response.status_code
print(f"{title}: {status}")
fact_sheets.append({"number": num, "title": title, "url": url, "status": status})
Reading room addresses and hours of opening: 200 Using our collection: 404 Addresses of Australian archival institutions: 404 Reading room rules: 404 What are archives?: 404 Archival terms: 200 The Commonwealth Record Series (CRS) system: 200 Citing archival records: 200 Copyright: 200 Searching for records: 404 Access to records under the Archives Act: 200 Viewing records in the reading room: 404 What to do if we refuse you access: 200 RecordSearch: an overview: 404 Keyword searching in RecordSearch Advanced search screens: 404 Release of records containing personal information: 200 Service guidelines for the National Reference Service: 404 NameSearch: 200 PhotoSearch: 404 Parliamentary Papers: 404 Commonwealth of Australia Gazettes: 200 Customs House, Sydney: 200 Coastal fortifications in New South Wales: 404 Commonwealth Film Unit: 404 The wine industry in South Australia: 404 Tasmanian railways: 404 Australia First Movement: 404 Commonwealth banking policy: 404 Navy service records: 404 Navy crew and ships records: 404 RAAF service records: 404 Security intelligence records held in Canberra: 200 Cabinet records: 404 Administration of the Australian Capital Territory: 404 Military records held in Hobart: 404 Maritime records held in Hobart: 200 Passenger records held in Canberra: 200 Civilian service in World War II: 404 Research agents – Canberra: 404 Research agents – Sydney: 404 Research agents – Brisbane: 404 Research agents – Adelaide and Darwin: 404 Research agents – Melbourne and Hobart: 404 Research agents – Perth: 404 Why we refuse access: 200 Australian Overseas Information Service photographs: 404 Papua New Guinea patrol reports: 404 D Notices: 200 Post Office records: 200 Copying charges: 200 Exempt information in ASIO records: 404 Personal information in ASIO records: 404 Veterans' case files: 200 Fremantle Harbour: 404 Passenger records held in Perth: 200 Melbourne Olympics, 1956: 404 World War I internee, alien and POW records held in Canberra: 404 World War II internee, alien and POW records held in Canberra: 404 Design and development of the national capital: 404 World War II war crimes: 404 Indonesian independence: 404 War service information: 404 Passenger records held in Sydney: 200 Customs shipping records held in Sydney: 404 Migrant selection documents held in Canberra: 404 Boer War records: 404 Naturalisation records held in Canberra: 200 ASIO files on writers and literary groups: 404 Prime ministers of Australia: 404 Prime Minister Joseph Cook: 404 Prime Minister William Morris Hughes: 404 Prime Minister Stanley Melbourne Bruce: 404 Prime Minister James Henry Scullin: 404 Prime Minister Joseph Aloysius Lyons: 404 Prime Minister Earle Christmas Grafton Page: 404 Prime Minister Robert Gordon Menzies: 404 Prime Minister Arthur William Fadden: 404 Prime Minister John Joseph Ambrose Curtin: 404 Prime Minister Francis Michael Forde: 404 Prime Minister Joseph Benedict Chifley: 404 Prime Minister Harold Edward Holt: 404 Prime Minister John McEwen: 404 Prime Minister John Grey Gorton: 404 Family history sources held in Canberra: 404 Family history sources held in Adelaide: 404 Australia and the United Nations: 404 Births, deaths and marriages: 404 Cyclones and the Northern Territory: 404 Coastal fortifications in South Australia: 404 Customs houses in South Australia: 404 Customs House, Port Adelaide, South Australia: 404 Excise control of distilled products in South Australia: 404 Walter Burley Griffin and the design of Canberra: 404 J T Lang and Lang Labor: 200 Regulation of beer and brewing in South Australia: 404 Sir Frederick Shedden and the Shedden collection: 404 Records relating to Italian migration held in Sydney: 404 World War II internee, alien and POW records held in Sydney: 404 The Australian flag: 200 The Cocos (Keeling) Islands: 404 Commonwealth electoral rolls held in Perth: 404 Copyright records: 404 World War I internee, alien and POW records held in Adelaide: 404 World War II internee, alien and POW records held in Adelaide: 404 The Pastoral industry in the Northern Territory: 404 Building the provisional Parliament House: 404 When to use the Freedom of Information, Archives and Privacy Acts: 200 The sinking of HMAS Sydney, November 1941: 200 Royal Commission into Aboriginal Deaths in Custody: 200 Aboriginal and Torres Strait Islander people: 404 Memorandum of Understanding with Northern Territory Aboriginal people: 404 Introducing television to Australia, 1956: 404 Guides to the collection: 404 Australia's involvement in the Vietnam War: 200 Computer resources in reading rooms: 404 Commonwealth electoral rolls held in Brisbane: 404 Bankruptcy records held in Sydney: 404 General Sir John Monash: 404 Lighthouse records held in Hobart: 404 Records of British migrants held in Canberra: 404 Child migration to Australia: 200 Radar research in Australia during World War II: 404 Radar production and use during World War II: 404 War Cabinet records: 404 Cabinet notebooks: 404 British nuclear tests at Maralinga: 404 The Royal Commission on Espionage, 1954–55: 404 Posters: 200 World War ll Army pay files held in Adelaide: 404 Defence and service records held in Melbourne: 404 Colonial defence personnel records held in Melbourne: 404 Army administrative records held in Melbourne: 404 Army service records: 404 Navy administrative records held in Melbourne: 404 Navy service records held in Melbourne: 404 Royalty and Australian society: 404 Cockatoo Island Dockyard: 404 Stevedoring industry: 404 Canberra air disaster, 1940: 404 North Head Quarantine Station, Sydney: 404 Harold Holt's disappearance, 1967: 200 Albert Namatjira: 404 Jessie Sinclair Litchfield: 404 Child migrant records held in Sydney: 404 Records of Papua New Guinea, 1883–1942: 200 Sound collections held in Sydney: 404 The 1967 referendum: 200 Bishop Francis Xavier Gsell: 404 Army and RAAF pay records held in Perth: 404 ABC Talks Department scripts: 404 External Affairs cables: 404 Edward John Connellan and Connellan Airways: 403 Records of Dutch migration held in Sydney: 404 Christmas Island: 404 Foundation of the State of Israel, 1946: 404 Reverend John Flynn and the Australian Inland Mission: 404 Universal military training in Australia, 1911–29: 200 Conscription referendums, 1916 and 1917: 200 National Service and war, 1939–45: 200 National Service, 1951–59: 404 National Service, 1965–72: 200 Royal Military College, Duntroon: 404 Government House, Canberra: 404 Mount Stromlo Observatory: 404 Gorman House, Canberra: 404 Migrant hostels in New South Wales, 1946–78: 200 World War I internee, alien and POW records held in Sydney: 404 Passenger records held in Melbourne: 200 Security intelligence records in Melbourne: 404 East Block building, Canberra: 404 Bringing Them Home name index: 200 Cyclone Tracy, Darwin: 200 World War I and World War II service records: 200 Commonwealth Reconstruction Training Scheme (CRTS) administrative records: 404 Commonwealth Reconstruction Training Scheme (CRTS) applicants and trainees: 404 Wartime internee, alien and POW records held in Perth: 200 Civil Constructional Corps records held in Perth: 404 Civil Alien Corps records held in Perth: 404 New Guard Movement, 1931–35: 404 Passenger records held in Hobart: 404 Migrant selection documents held in Perth: 404 Alien registration records held in Perth: 404 Citizenship in Australia: 200 Empire Games, Sydney, 1938: 403 General Post Office, Sydney: 200 Passenger records held in Brisbane: 404 Aerial photographs: 404 Japanese midget submarine attacks on Sydney, 1942: 200 Addresses of other national archives: 404 Australian Antarctic exploration and research: 404 The bombing of Darwin: 200 Sir Charles Kingsford Smith: 404 Photographs relating to Sir Charles Kingsford Smith: 404 Cowra breakout, 1944: 200 Army Inventions Directorate, 1942–46: 404 Beginning your family history research: 404 Tracing ancestors in the National Archives: 404 Tracing ancestors beyond the National Archives: 404 The House of Representatives Standing Committee of Privileges: 404 The Browne-Fitzpatrick privilege case, 1955: 404 Memorandum of Understanding with the Victorian Aboriginal Child Care Agency: 404 Records relating to Italian migration held in Perth: 404 Research Agents – overseas institutions: 404 Memorandum of Understanding with South Australian Indigenous people: 404 Prime Minister Edmund Barton: 404 Prime Minister Alfred Deakin: 404 Prime Minister John Christian Watson: 404 Prime Minister George Houstoun Reid: 404 Prime Minister Andrew Fisher: 404 Prime Minister William McMahon: 404 Prime Minister Edward Gough Whitlam: 404 The Jewish experience in Australia: 404 The National Archives collecting policy: 404 Special access: 404 Passenger arrivals index: 200 High Court of Australia: 404 Mildenhall photographic collection: 404 Migrant selection documents in Adelaide: 200 The Wave Hill walk-off: 200 Charles Nelson Perkins: 404 Custom House, Brisbane: 404 Immigration records: 404 Torrens Island Quarantine Station, South Australia: 200 Access to damaged, fragile or contaminated records: 404 Using cameras in the reading room: 404 Neville Bonner: 404 Industrial relations records held in Melbourne: 404 Lighthouse records held in Brisbane: 404 United States forces in Queensland, 1941–45: 200 Francis Edgar Williams, anthropologist of Papua: 404 Records relating to Italian migration held in Brisbane: 404 International Women's Year, 1975: 404 The 'Balibo affair', East Timor, October 1975: 404 The loans affair, 1974–75: 404 The dismissal, 1975: 404 John Robert Kerr, Governor-General of Australia, 1974–77: 404 Prime Minister John Malcolm Fraser: 404 The fall of Saigon, 1975: 404 Industrial development in Australia after World War II: 404 Patent, trademark and design records in Brisbane: 404 Cabinet records of the Fraser government, 1975–83: 404 Australia's diplomatic relations with China: 403 Daniel Mannix, Catholic Archbishop of Melbourne: 200 The National Archives digitisation service: 404 Albert Hall, Canberra: 404 Australia’s national anthem: 200 Tobacco advertising ban in Australia: 404 Australian Atomic Energy Commission: 404 The Immigration Photographic Archive: 404 Australia and the issue of apartheid in sport: 404 Passenger records held in Adelaide: 200 Official access: 404 Torres Strait Treaty, 1978: 404 South Australian lighthouse records: 404 South Australian maritime records: 404 Independence of Papua New Guinea, 1975: 404 Royal Commission on Intelligence and Security: 404 Independence of Zimbabwe: 404 Finding records relating to an Indigenous person: 404 Patent records held in Canberra: 404 The sinking of the Montevideo Maru: 404 Robert James Lee Hawke: 404 Aboriginal petitions: 404 South Sea Islanders: 404 Indigenous family history beyond the National Archives: 404 Downer family collection: 404
df = pd.DataFrame(fact_sheets)
Let's break down the results by HTTP status code.
df["status"].value_counts()
404 207 200 56 403 3 Name: status, dtype: int64
print(f"{(207 + 3) / (207 + 56 + 3):.2%} of fact sheets are kaput!")
78.95% of fact sheets are kaput!
df.loc[df["status"] == 200]
number | title | url | status | |
---|---|---|---|---|
0 | 1 | Reading room addresses and hours of opening | http://naa.gov.au/collection/fact-sheets/fs01.... | 200 |
5 | 5 | Archival terms | http://naa.gov.au/collection/fact-sheets/fs05.... | 200 |
6 | 6 | The Commonwealth Record Series (CRS) system | http://naa.gov.au/collection/fact-sheets/fs06.... | 200 |
7 | 7 | Citing archival records | http://naa.gov.au/collection/fact-sheets/fs07.... | 200 |
8 | 8 | Copyright | http://naa.gov.au/collection/fact-sheets/fs08.... | 200 |
10 | 10 | Access to records under the Archives Act | http://naa.gov.au/collection/fact-sheets/fs10.... | 200 |
12 | 12 | What to do if we refuse you access | http://naa.gov.au/collection/fact-sheets/fs12.... | 200 |
15 | 15 | Release of records containing personal informa... | http://naa.gov.au/collection/fact-sheets/fs15.... | 200 |
17 | 18 | NameSearch | http://naa.gov.au/collection/fact-sheets/fs18.... | 200 |
20 | 22 | Commonwealth of Australia Gazettes | http://naa.gov.au/collection/fact-sheets/fs22.... | 200 |
21 | 23 | Customs House, Sydney | http://naa.gov.au/collection/fact-sheets/fs23.... | 200 |
31 | 33 | Security intelligence records held in Canberra | http://naa.gov.au/collection/fact-sheets/fs33.... | 200 |
35 | 37 | Maritime records held in Hobart | http://naa.gov.au/collection/fact-sheets/fs37.... | 200 |
36 | 38 | Passenger records held in Canberra | http://naa.gov.au/collection/fact-sheets/fs38.... | 200 |
44 | 46 | Why we refuse access | http://naa.gov.au/collection/fact-sheets/fs46.... | 200 |
47 | 49 | D Notices | http://naa.gov.au/collection/fact-sheets/fs49.... | 200 |
48 | 50 | Post Office records | http://naa.gov.au/collection/fact-sheets/fs50.... | 200 |
49 | 51 | Copying charges | http://naa.gov.au/collection/fact-sheets/fs51.... | 200 |
52 | 54 | Veterans' case files | http://naa.gov.au/collection/fact-sheets/fs54.... | 200 |
54 | 56 | Passenger records held in Perth | http://naa.gov.au/collection/fact-sheets/fs56.... | 200 |
62 | 64 | Passenger records held in Sydney | http://naa.gov.au/collection/fact-sheets/fs64.... | 200 |
66 | 68 | Naturalisation records held in Canberra | http://naa.gov.au/collection/fact-sheets/fs68.... | 200 |
93 | 96 | J T Lang and Lang Labor | http://naa.gov.au/collection/fact-sheets/fs96.... | 200 |
98 | 102 | The Australian flag | http://naa.gov.au/collection/fact-sheets/fs102... | 200 |
106 | 110 | When to use the Freedom of Information, Archiv... | http://naa.gov.au/collection/fact-sheets/fs110... | 200 |
107 | 111 | The sinking of HMAS Sydney, November 1941 | http://naa.gov.au/collection/fact-sheets/fs111... | 200 |
108 | 112 | Royal Commission into Aboriginal Deaths in Cus... | http://naa.gov.au/collection/fact-sheets/fs112... | 200 |
113 | 117 | Australia's involvement in the Vietnam War | http://naa.gov.au/collection/fact-sheets/fs117... | 200 |
120 | 124 | Child migration to Australia | http://naa.gov.au/collection/fact-sheets/fs124... | 200 |
127 | 131 | Posters | http://naa.gov.au/collection/fact-sheets/fs131... | 200 |
140 | 144 | Harold Holt's disappearance, 1967 | http://naa.gov.au/collection/fact-sheets/fs144... | 200 |
144 | 148 | Records of Papua New Guinea, 1883–1942 | http://naa.gov.au/collection/fact-sheets/fs148... | 200 |
146 | 150 | The 1967 referendum | http://naa.gov.au/collection/fact-sheets/fs150... | 200 |
156 | 160 | Universal military training in Australia, 1911–29 | http://naa.gov.au/collection/fact-sheets/fs160... | 200 |
157 | 161 | Conscription referendums, 1916 and 1917 | http://naa.gov.au/collection/fact-sheets/fs161... | 200 |
158 | 162 | National Service and war, 1939–45 | http://naa.gov.au/collection/fact-sheets/fs162... | 200 |
160 | 164 | National Service, 1965–72 | http://naa.gov.au/collection/fact-sheets/fs164... | 200 |
165 | 170 | Migrant hostels in New South Wales, 1946–78 | http://naa.gov.au/collection/fact-sheets/fs170... | 200 |
167 | 172 | Passenger records held in Melbourne | http://naa.gov.au/collection/fact-sheets/fs172... | 200 |
170 | 175 | Bringing Them Home name index | http://naa.gov.au/collection/fact-sheets/fs175... | 200 |
171 | 176 | Cyclone Tracy, Darwin | http://naa.gov.au/collection/fact-sheets/fs176... | 200 |
172 | 177 | World War I and World War II service records | http://naa.gov.au/collection/fact-sheets/fs177... | 200 |
175 | 180 | Wartime internee, alien and POW records held i... | http://naa.gov.au/collection/fact-sheets/fs180... | 200 |
182 | 187 | Citizenship in Australia | http://naa.gov.au/collection/fact-sheets/fs187... | 200 |
184 | 189 | General Post Office, Sydney | http://naa.gov.au/collection/fact-sheets/fs189... | 200 |
187 | 192 | Japanese midget submarine attacks on Sydney, 1942 | http://naa.gov.au/collection/fact-sheets/fs192... | 200 |
190 | 195 | The bombing of Darwin | http://naa.gov.au/collection/fact-sheets/fs195... | 200 |
193 | 198 | Cowra breakout, 1944 | http://naa.gov.au/collection/fact-sheets/fs198... | 200 |
214 | 220 | Passenger arrivals index | http://naa.gov.au/collection/fact-sheets/fs220... | 200 |
217 | 223 | Migrant selection documents in Adelaide | http://naa.gov.au/collection/fact-sheets/fs223... | 200 |
218 | 224 | The Wave Hill walk-off | http://naa.gov.au/collection/fact-sheets/fs224... | 200 |
222 | 228 | Torrens Island Quarantine Station, South Austr... | http://naa.gov.au/collection/fact-sheets/fs228... | 200 |
228 | 234 | United States forces in Queensland, 1941–45 | http://naa.gov.au/collection/fact-sheets/fs234... | 200 |
242 | 248 | Daniel Mannix, Catholic Archbishop of Melbourne | http://naa.gov.au/collection/fact-sheets/fs248... | 200 |
245 | 251 | Australia’s national anthem | http://naa.gov.au/collection/fact-sheets/fs251... | 200 |
250 | 256 | Passenger records held in Adelaide | http://naa.gov.au/collection/fact-sheets/fs256... | 200 |
df.to_csv("data/fact_sheets.csv", index=False)