From 6873cab9554efadd13ec6988cc693c28044ca07b Mon Sep 17 00:00:00 2001 From: stijndcl Date: Sun, 19 Jun 2022 00:23:25 +0200 Subject: [PATCH] Ufora announcements --- ...efe_add_option_to_disable_announcement_.py | 32 ++++ database/crud/ufora_announcements.py | 17 ++ database/engine.py | 2 +- database/models.py | 11 +- didier/cogs/tasks.py | 49 ++++++ didier/data/embeds/__init__.py | 0 didier/data/embeds/ufora/__init__.py | 0 didier/data/embeds/ufora/announcements.py | 145 ++++++++++++++++++ pyproject.toml | 8 +- requirements-dev.txt | 3 +- requirements.txt | 3 + settings.py | 7 + 12 files changed, 267 insertions(+), 10 deletions(-) create mode 100644 alembic/versions/d3cd92cb9efe_add_option_to_disable_announcement_.py create mode 100644 database/crud/ufora_announcements.py create mode 100644 didier/cogs/tasks.py create mode 100644 didier/data/embeds/__init__.py create mode 100644 didier/data/embeds/ufora/__init__.py create mode 100644 didier/data/embeds/ufora/announcements.py diff --git a/alembic/versions/d3cd92cb9efe_add_option_to_disable_announcement_.py b/alembic/versions/d3cd92cb9efe_add_option_to_disable_announcement_.py new file mode 100644 index 0000000..99d8663 --- /dev/null +++ b/alembic/versions/d3cd92cb9efe_add_option_to_disable_announcement_.py @@ -0,0 +1,32 @@ +"""Add option to disable announcement fetching for courses + +Revision ID: d3cd92cb9efe +Revises: 9e8ce58c0a26 +Create Date: 2022-06-18 00:36:00.484627 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "d3cd92cb9efe" +down_revision = "9e8ce58c0a26" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("ufora_courses", schema=None) as batch_op: + batch_op.add_column(sa.Column("log_announcements", sa.Boolean(), nullable=False, server_default=sa.false())) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("ufora_courses", schema=None) as batch_op: + batch_op.drop_column("log_announcements") + + # ### end Alembic commands ### diff --git a/database/crud/ufora_announcements.py b/database/crud/ufora_announcements.py new file mode 100644 index 0000000..e72b344 --- /dev/null +++ b/database/crud/ufora_announcements.py @@ -0,0 +1,17 @@ +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from database.models import UforaCourse, UforaAnnouncement + + +async def get_courses_with_announcements(session: AsyncSession) -> list[UforaCourse]: + """Get all courses where announcements are enabled""" + query = select(UforaCourse).where(UforaCourse.log_announcements) + return (await session.execute(query)).scalars().all() + + +async def create_new_announcement(session: AsyncSession, announcement_id: int, course: UforaCourse): + """Add a new announcement to the database""" + new_announcement = UforaAnnouncement(announcement_id=announcement_id, course=course) + session.add(new_announcement) + await session.commit() diff --git a/database/engine.py b/database/engine.py index 2c1815f..70c70fe 100644 --- a/database/engine.py +++ b/database/engine.py @@ -7,7 +7,7 @@ from sqlalchemy.orm import sessionmaker import settings # Run local tests against SQLite instead of Postgres -if settings.DB_TEST_SQLITE: +if settings.TESTING and settings.DB_TEST_SQLITE: engine = create_async_engine( URL.create( drivername="sqlite+aiosqlite", diff --git a/database/models.py b/database/models.py index 3d778eb..142ba8b 100644 --- a/database/models.py +++ b/database/models.py @@ -1,6 +1,6 @@ from __future__ import annotations -from sqlalchemy import Column, Integer, Text, ForeignKey +from sqlalchemy import Column, Integer, Text, ForeignKey, Boolean from sqlalchemy.orm import declarative_base, relationship Base = declarative_base() @@ -15,12 +15,13 @@ class UforaCourse(Base): name: str = Column(Text, nullable=False, unique=True) code: str = Column(Text, nullable=False, unique=True) year: int = Column(Integer, nullable=False) + log_announcements: bool = Column(Boolean, default=False, nullable=False) announcements: list[UforaAnnouncement] = relationship( - "UforaAnnouncement", back_populates="course", cascade="all, delete-orphan" + "UforaAnnouncement", back_populates="course", cascade="all, delete-orphan", lazy="selectin" ) aliases: list[UforaCourseAlias] = relationship( - "UforaCourseAlias", back_populates="course", cascade="all, delete-orphan" + "UforaCourseAlias", back_populates="course", cascade="all, delete-orphan", lazy="selectin" ) @@ -33,7 +34,7 @@ class UforaCourseAlias(Base): alias: str = Column(Text, nullable=False, unique=True) course_id: int = Column(Integer, ForeignKey("ufora_courses.course_id")) - course: UforaCourse = relationship("UforaCourse", back_populates="aliases", uselist=False) + course: UforaCourse = relationship("UforaCourse", back_populates="aliases", uselist=False, lazy="selectin") class UforaAnnouncement(Base): @@ -44,4 +45,4 @@ class UforaAnnouncement(Base): announcement_id = Column(Integer, primary_key=True) course_id = Column(Integer, ForeignKey("ufora_courses.course_id")) - course: UforaCourse = relationship("UforaCourse", back_populates="announcements", uselist=False) + course: UforaCourse = relationship("UforaCourse", back_populates="announcements", uselist=False, lazy="selectin") diff --git a/didier/cogs/tasks.py b/didier/cogs/tasks.py new file mode 100644 index 0000000..45e5ada --- /dev/null +++ b/didier/cogs/tasks.py @@ -0,0 +1,49 @@ +import traceback + +from discord.ext import commands, tasks + +import settings +from didier import Didier +from didier.data.embeds.ufora.announcements import fetch_ufora_announcements + + +# TODO task to clean up old announcements? (> 1 week) +class Tasks(commands.Cog): + """Task loops that run periodically""" + + client: Didier + + def __init__(self, client: Didier): + self.client = client + + # Only pull announcements if a token was provided + if settings.UFORA_RSS_TOKEN is not None and settings.UFORA_ANNOUNCEMENTS_CHANNEL is not None: + self.pull_ufora_announcements.start() # pylint: disable=no-member + + @tasks.loop(minutes=10) + async def pull_ufora_announcements(self): + """Task that checks for new Ufora announcements & logs them in a channel""" + # In theory this shouldn't happen but just to please Mypy + if settings.UFORA_RSS_TOKEN is None or settings.UFORA_ANNOUNCEMENTS_CHANNEL is None: + return + + announcements_channel = self.client.get_channel(settings.UFORA_ANNOUNCEMENTS_CHANNEL) + announcements = await fetch_ufora_announcements(self.client.db_session) + + for announcement in announcements: + await announcements_channel.send(embed=announcement.to_embed()) + + @pull_ufora_announcements.before_loop + async def _before_ufora_announcements(self): + """Don't try to get announcements if the bot isn't ready yet""" + await self.client.wait_until_ready() + + @pull_ufora_announcements.error + async def _on_announcements_error(self, error: BaseException): + """Error handler for the Ufora Announcements task""" + print("".join(traceback.format_exception(type(error), error, error.__traceback__))) + + +async def setup(client: Didier): + """Load the cog""" + await client.add_cog(Tasks(client)) diff --git a/didier/data/embeds/__init__.py b/didier/data/embeds/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/didier/data/embeds/ufora/__init__.py b/didier/data/embeds/ufora/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/didier/data/embeds/ufora/announcements.py b/didier/data/embeds/ufora/announcements.py new file mode 100644 index 0000000..47a4755 --- /dev/null +++ b/didier/data/embeds/ufora/announcements.py @@ -0,0 +1,145 @@ +import re +from dataclasses import dataclass, field +from datetime import datetime +from typing import Optional + +import discord +import feedparser +import pytz +from markdownify import markdownify as md +from sqlalchemy.ext.asyncio import AsyncSession + +import settings +from database.crud import ufora_announcements as crud +from database.models import UforaCourse + + +@dataclass +class UforaNotification: + """A single notification from Ufora""" + + content: dict + course: UforaCourse + notification_id: Optional[int] = None + course_id: Optional[int] = None + + _view_url: str = field(init=False) + _title: str = field(init=False) + _description: str = field(init=False) + _published: str = field(init=False) + + def __post_init__(self): + self._view_url = self._create_url() + self._title = self._clean_content(self.content["title"]) + self._description = self._get_description() + self._published = self._get_published() + + def to_embed(self) -> discord.Embed: + """Turn the notification into an embed""" + embed = discord.Embed(colour=discord.Colour.from_rgb(30, 100, 200)) + + embed.set_author(name=self.course.name) + embed.title = self._title + embed.url = self._view_url + embed.description = self._description + embed.set_footer(text=self._published) + + return embed + + def get_id(self) -> int: + """Parse the id out of the notification""" + return int(self.notification_id) if self.notification_id is not None else self.content["id"] + + def _create_url(self): + if self.notification_id is None or self.course_id is None: + return self.content["link"] + + return f"https://ufora.ugent.be/d2l/le/news/{self.course_id}/{self.notification_id}/view?ou={self.course_id}" + + def _get_description(self): + desc = self._clean_content(self.content["summary"]) + + if len(desc) > 4096: + return desc[:4093] + "..." + + return desc + + def _clean_content(self, text: str): + # Escape *-characters because they mess up the layout + text = text.replace("*", "\\*") + return md(text) + + def _get_published(self) -> str: + # Datetime is unable to parse the timezone because it's useless + # We will hereby cut it out and pray the timezone will always be UTC+0 + published = self.content["published"].rsplit(" ", 1)[0] + time_string = "%a, %d %b %Y %H:%M:%S" + dt = datetime.strptime(published, time_string).astimezone(pytz.timezone("Europe/Brussels")) + + # Apply timezone offset in a hacky way + offset = dt.utcoffset() + if offset is not None: + dt += offset + + # TODO + return "Placeholder :) TODO make the functions to format this" + + +def parse_ids(url: str) -> Optional[tuple[int, int]]: + """Parse the notification & course id out of a notification url""" + match = re.search(r"\d+-\d+$", url) + + if not match: + return None + + spl = match[0].split("-") + return int(spl[0]), int(spl[1]) + + +async def fetch_ufora_announcements(session: AsyncSession) -> list[UforaNotification]: + """Fetch all new announcements""" + notifications: list[UforaNotification] = [] + + # No token provided, don't fetch announcements + if settings.UFORA_RSS_TOKEN is None: + return notifications + + courses = await crud.get_courses_with_announcements(session) + + for course in courses: + course_announcement_ids = list(map(lambda announcement: announcement.announcement_id, course.announcements)) + + course_url = ( + f"https://ufora.ugent.be/d2l/le/news/rss/{course.course_id}/course?token={settings.UFORA_RSS_TOKEN}" + ) + + # Get the updated feed + feed = feedparser.parse(course_url) + + # Remove old notifications + fresh_feed: list[dict] = [] + for entry in feed["entries"]: + parsed = parse_ids(entry["id"]) + if parsed is None: + continue + + if parsed[0] not in course_announcement_ids: + fresh_feed.append(entry) + + if fresh_feed: + for item in fresh_feed: + # Parse id's out + # Technically this can't happen but Mypy angry + parsed = parse_ids(item["id"]) + + if parsed is None: + continue + + # Create a new notification + notification_id, course_id = parsed + notifications.append(UforaNotification(item, course, notification_id, course_id)) + + # Create new db entry + await crud.create_new_announcement(session, notification_id, course) + + return notifications diff --git a/pyproject.toml b/pyproject.toml index a1a3d6a..bfc1ced 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,23 +6,25 @@ plugins = [ "sqlalchemy.ext.mypy.plugin" ] [[tool.mypy.overrides]] -module = "discord.*" +module = ["discord.*", "feedparser.*", "markdownify.*"] ignore_missing_imports = true [tool.pylint.master] disable = [ "missing-module-docstring", "too-few-public-methods", - "too-many-arguments" + "too-many-arguments", + "too-many-instance-attributes" ] [tool.pylint.format] max-line-length = 120 -good-names = ["i"] +good-names = ["i", "dt"] [tool.pytest.ini_options] asyncio_mode = "auto" env = [ + "TESTING = true", "DB_NAME = didier_action", "DB_USERNAME = postgres", "DB_HOST = localhost", diff --git a/requirements-dev.txt b/requirements-dev.txt index 75ae1f9..ef7679c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,4 +5,5 @@ pylint==2.14.1 pytest==7.1.2 pytest-asyncio==0.18.3 pytest-env==0.6.2 -sqlalchemy2-stubs==0.0.2a23 \ No newline at end of file +sqlalchemy2-stubs==0.0.2a23 +types-pytz==2021.3.8 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 499739d..1b4240a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,7 @@ asyncpg==0.25.0 # Dev version of dpy git+https://github.com/Rapptz/discord.py environs==9.5.0 +feedparser==6.0.10 +markdownify==0.11.2 +pytz==2022.1 sqlalchemy[asyncio]==1.4.37 diff --git a/settings.py b/settings.py index cd7e541..ee0c448 100644 --- a/settings.py +++ b/settings.py @@ -1,3 +1,5 @@ +from typing import Optional + from environs import Env # Read the .env file (if present) @@ -7,6 +9,7 @@ env.read_env() """General config""" SANDBOX: bool = env.bool("SANDBOX", True) LOGFILE: str = env.str("LOGFILE", "didier.log") +TESTING: bool = env.bool("TESTING", False) """Database""" DB_NAME: str = env.str("DB_NAME", "didier") @@ -21,3 +24,7 @@ DISCORD_TOKEN: str = env.str("DISC_TOKEN") DISCORD_READY_MESSAGE: str = env.str("DISC_READY_MESSAGE", "I'M READY I'M READY I'M READY") DISCORD_STATUS_MESSAGE: str = env.str("DISC_STATUS_MESSAGE", "with your Didier Dinks.") DISCORD_TEST_GUILDS: list[int] = env.list("DISC_TEST_GUILDS", [], subcast=int) +UFORA_ANNOUNCEMENTS_CHANNEL: Optional[int] = env.int("UFORA_ANNOUNCEMENTS_CHANNEL", None) + +"""API Keys""" +UFORA_RSS_TOKEN: Optional[str] = env.str("UFORA_RSS_TOKEN", None)