def search_experiences(): print('Experience Vaults') experiences = {} soup_text = [] SEARCH_URL = "https://erowid.org/experiences/subs/exp_" substance = input('Search>').strip().lower() substance = substance[0].upper() + substance[1:] category = {'General':SEARCH_URL+substance+'_General.shtml','First Times':SEARCH_URL+substance+'_First_Times.shtml', 'Combonations':SEARCH_URL+substance+'_Combonations.shtml',\ 'Retrospective/Summary':SEARCH_URL+substance+'_Retrospective_I_Summary.shtml', 'Preperation/Recipes':SEARCH_URL+substance+'_Preperation_I_Recipes.shtml',\ 'Difficult Experiences':SEARCH_URL+substance+'_Difficult_Experiences.shtml', 'Bad Trips':SEARCH_URL+substance+'_Bad_Trips.shtml', 'Health Problems': SEARCH_URL+substance+'_Health_Problems.shtml',\ 'Train Wrecks and Distaster Trips':SEARCH_URL+substance+'_Train_Wrecks_Trip_Disasters.shtml', 'Addiction and Habituation':SEARCH_URL+substance+'_Addiction_Habituation.shtml',\ 'Glowing Experiences':SEARCH_URL+substance+'_Glowing_Experiences.shtml', 'Msytical Experiences':SEARCH_URL+substance+'_Mystical_Experiences.shtml', 'Health Benfits':SEARCH_URL+substance+'_Health_Benefits.shtml',\ 'Families':SEARCH_URL+substance+'_Families.shtml', 'Medical Use':SEARCH_URL+substance+'_Medical_Use.shtml','What Was In That?':SEARCH_URL+substance+'_What_Was_in_That.shtml'} x=1 for key in category: print('[+]',x,'.', key) x+=1 cat = input('Category>').strip() if cat == 'exit:': sys.exit() if cat == "main": menu() #Grab page data -- value of key URL = (category[cat]) data = requests.get(URL) content = data.text search_soup = BeautifulSoup(content, 'html.parser') x = 0 for link in search_soup.find_all('a'): print('[+]',x, link.text) experiences[x] = link.get('href') x += 1 choice = input('Choice>').strip() data = requests.get('https://erowid.org'+ str(experiences[int(choice)])) content = data.text search_soup = BeautifulSoup(content, 'html.parser') #for comment in search_soup.find_all(string=lambda text:isinstance(text,Comment)): # print(comment) soup_text.append(str(search_soup)) #experience = str(re.search(r"(.*)",str(soup_text[0]))) for index in soup_text: print(soup_text) experience = re.search('(.*)', soup_text[0]) print(experience)