diff --git a/cogs/google.py b/cogs/google.py index 446d6d8..87b1650 100644 --- a/cogs/google.py +++ b/cogs/google.py @@ -2,7 +2,7 @@ import discord from discord.ext import commands from decorators import help from enums.help_categories import Category -from functions.scrapers.google import google_search +from functions.scraping import google_search class Google(commands.Cog): diff --git a/cogs/randomCog.py b/cogs/random.py similarity index 100% rename from cogs/randomCog.py rename to cogs/random.py diff --git a/cogs/tasks.py b/cogs/tasks.py index 7f17b2d..2c4818a 100644 --- a/cogs/tasks.py +++ b/cogs/tasks.py @@ -5,7 +5,7 @@ from enums.numbers import Numbers from functions import timeFormatters from functions.config import config from functions.database import currency, poke, prison, birthdays, stats -from functions.scrapers.sporza import getMatchweek +from functions.scraping import getMatchweek from functions import ufora_notifications import json import random @@ -202,12 +202,6 @@ class Tasks(commands.Cog): if (not category["weekends"]) and weekday > 4: continue - # Create embed once because this can be heavy - if "embed" in category: - embed = category["embed"]() - else: - embed = None - for user in category["users"]: userInstance = self.client.get_user(user) @@ -219,7 +213,7 @@ class Tasks(commands.Cog): if "embed" not in category: await userInstance.send(random.choice(category["messages"])) else: - await userInstance.send(random.choice(category["messages"]), embed=embed) + await userInstance.send(random.choice(category["messages"]), embed=category["embed"]) with open("files/lastTasks.json", "w") as fp: lastTasks["remind"] = round(time.time()) @@ -234,10 +228,6 @@ class Tasks(commands.Cog): """ Task that checks the current JPL matchweek & changes the dict value """ - # Don't run this when testing - if self.client.user.id != int(constants.didierId): - return - matchweek = getMatchweek() if matchweek is None: diff --git a/data/remind.py b/data/remind.py index 39f6dd3..bfe4768 100644 --- a/data/remind.py +++ b/data/remind.py @@ -1,5 +1,4 @@ -from data import schedule -from functions import les, config +from functions import les from functions.database import remind @@ -13,11 +12,13 @@ class Reminders: self._les = [int(user[0]) for user in rows if user[2]] self._lesMessages = ["Lessenrooster voor vandaag:"] - self.les = {"users": self._les, "messages": self._lesMessages, "embed": self.lesEmbed, "weekends": False, "disabled": True} + self.les = {"users": self._les, "messages": self._lesMessages, "embed": self.lesEmbed(), "weekends": False, "disabled": True} self.categories = [self.nightly, self.les] def lesEmbed(self): - dt = les.find_target_date() - s = schedule.Schedule(dt, int(config.get("year")), int(config.get("semester"))) - return s.create_schedule().to_embed() + day, dayDatetime, semester, year = les.parseArgs([])[1:] + + schedule = les.getSchedule(semester, year) + + return les.createEmbed(day, dayDatetime, semester, year, schedule) \ No newline at end of file diff --git a/functions/football.py b/functions/football.py index 092a90c..64941ad 100644 --- a/functions/football.py +++ b/functions/football.py @@ -1,7 +1,7 @@ from enum import Enum from attr import dataclass, field from functions.timeFormatters import fromString -from functions.scrapers.sporza import getJPLMatches, getJPLTable +from functions.scraping import getJPLMatches, getJPLTable from functions.stringFormatters import leadingZero from datetime import datetime import tabulate diff --git a/functions/les.py b/functions/les.py index 628a155..c3a7cfb 100644 --- a/functions/les.py +++ b/functions/les.py @@ -3,7 +3,7 @@ from functions.timeFormatters import dateTimeNow, weekdayToInt, forward_to_weekd from typing import Optional -def find_target_date(arg: Optional[str] = None) -> datetime: +def find_target_date(arg: Optional[str]) -> datetime: """ Find the requested date out of the user's arguments """ diff --git a/functions/scrapers/__init__.py b/functions/scrapers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/functions/scrapers/google.py b/functions/scrapers/google.py deleted file mode 100644 index 4b7aefa..0000000 --- a/functions/scrapers/google.py +++ /dev/null @@ -1,38 +0,0 @@ -from bs4 import BeautifulSoup -from requests import get -from urllib.parse import urlencode - - -def google_search(query): - """ - Function to get Google search results - """ - headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36' - } - - query = urlencode({"q": query}) - - # Get 20 results in case some of them are None - resp = get("https://www.google.com/search?{}&num=20&hl=en".format(query), headers=headers) - - if resp.status_code != 200: - return None, resp.status_code - - bs = BeautifulSoup(resp.text, "html.parser") - - def getContent(element): - """ - Function to find links & titles in the HTML of a
element - """ - link = element.find("a", href=True) - title = element.find("h3") - - if link is None or title is None: - return None - - return link["href"], title.text - - divs = bs.find_all("div", attrs={"class": "g"}) - - return list(getContent(d) for d in divs), 200 diff --git a/functions/scrapers/sporza.py b/functions/scraping.py similarity index 58% rename from functions/scrapers/sporza.py rename to functions/scraping.py index 69a8633..7b642ec 100644 --- a/functions/scrapers/sporza.py +++ b/functions/scraping.py @@ -1,6 +1,45 @@ -from bs4 import BeautifulSoup import re + from requests import get +from urllib.parse import urlencode +from bs4 import BeautifulSoup + +# TODO add Football requests in here as well + + +def google_search(query): + """ + Function to get Google search results + """ + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36' + } + + query = urlencode({"q": query}) + + # Get 20 results in case some of them are None + resp = get("https://www.google.com/search?{}&num=20&hl=en".format(query), headers=headers) + + if resp.status_code != 200: + return None, resp.status_code + + bs = BeautifulSoup(resp.text, "html.parser") + + def getContent(element): + """ + Function to find links & titles in the HTML of a
element + """ + link = element.find("a", href=True) + title = element.find("h3") + + if link is None or title is None: + return None + + return link["href"], title.text + + divs = bs.find_all("div", attrs={"class": "g"}) + + return list(getContent(d) for d in divs), 200 def getMatchweek(): diff --git a/tests/functions/scrapers/__init__.py b/tests/functions/scrapers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/functions/scrapers/test_sporza.py b/tests/functions/scrapers/test_sporza.py deleted file mode 100644 index 744341f..0000000 --- a/tests/functions/scrapers/test_sporza.py +++ /dev/null @@ -1,13 +0,0 @@ -from functions.scrapers import sporza -import unittest - - -class TestSporzaScraper(unittest.TestCase): - def test_find_matchweek(self): - """ - This tests if the structure of the HTML is still what we expect it to be, - as Sporza changes it from time to time. - """ - # This will throw an error if the argument was not a proper integer - week = int(sporza.getMatchweek()) - self.assertGreater(week, 0)