from bs4 import BeautifulSoup ALL_SUBSTANCE = "https://www.erowid.org/experiences/exp_list.shtml" substance_list = [] data = requests.get(ALL_SUBSTANCE) content = data.content substance_soup = BeautifulSoup(content, 'html.parser') for link in substance_soup.findAll('a'): print(link)