Chewing_Bever/didier
0
0
Fork 0
mirror of https://github.com/stijndcl/didier.git synced 2026-04-07 15:48:29 +02:00
Code Issues Projects Releases Wiki Activity

Change discord version to 1.6.0, Google search (fixes #18)

Browse source
This commit is contained in:
Stijn De Clercq 2021-01-24 13:08:58 +01:00
parent 3a6e57db73
commit 83f28d9e21
4 changed files with 43 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

22
functions/scraping.py
View file

@ -15,18 +15,26 @@ def google_search(query):
query = urlencode({"q": query})
resp = get("https://www.google.com/search?{}&num=10&hl=en".format(query), headers=headers)
print("got here")
# Get 20 results in case some of them are None
resp = get("https://www.google.com/search?{}&num=20&hl=en".format(query), headers=headers)
if resp.status_code != 200:
return None, resp.status_code
bs = BeautifulSoup(resp.text, "html.parser")
def getContent(element):
link = element.find('a', href=True)
title = element.find('h3')
return link, title
link = element.find("a", href=True)
title = element.find("h3")
if link is None or title is None:
return None
sp = title.find("span")
if sp is None:
return None
return link["href"], sp.text
divs = bs.find_all("div", attrs={"class": "g"})
divs = bs.find_all('div', attrs={'class': 'g'})
print(divs)
return list(getContent(d) for d in divs), 200