Merge pull request #72 from stijndcl/flask-backend

Create basics for backend server, + small fixes

.gitignore

@@ -1,14 +1,11 @@
-files/status.txt
-files/readyMessage.txt
-files/client.txt
 files/lastTasks.json
 files/c4.json
 files/hangman.json
 files/stats.json
 files/lost.json
 files/locked.json
-files/database.json
 files/ufora_notifications.json
 .idea/
 __pycache__
-.env
+.env
+/venv/

backend/server.py

@@ -0,0 +1,82 @@
+from discord.ext import ipc
+from functions.database import custom_commands
+import json
+from quart import Quart, jsonify, request
+from quart_cors import cors
+from time import time
+
+
+app = Quart(__name__)
+# TODO allow_origin=re.compile(r"http://localhost:.*")
+#      needs higher Python & Quart version
+app = cors(app, allow_origin="*")
+app.config.from_object(__name__)
+
+
+ipc_client = ipc.Client(secret_key="SOME_SECRET_KEY")
+
+
+@app.route("/ping", methods=["GET"])
+async def ping():
+    """
+    Send a ping request, monitors bot latency, endpoint time, and PSQL latency
+    """
+    latency = await ipc_client.request("get_bot_latency")
+
+    return jsonify({"bot_latency": latency, "response_sent": time()})
+
+
+@app.route("/dm", methods=["POST"])
+async def send_dm():
+    """
+    Send a DM to the given user
+    """
+    data = json.loads((await request.body).decode('UTF-8'))
+
+    dm = await ipc_client.request(
+        "send_dm",
+        user=int(data["userid"]),
+        message=data.get("message")
+    )
+
+    return jsonify({"response": dm})
+
+
+@app.route("/custom", methods=["GET"])
+async def get_all_custom_commands():
+    """
+    Return a list of all custom commands in the bot
+    """
+    commands = custom_commands.get_all()
+
+    return jsonify(commands)
+
+
+@app.route("/custom/<command_id>")
+async def get_custom_command(command_id):
+    try:
+        command_id = int(command_id)
+    except ValueError:
+        # Id is not an int
+        return unprocessable_entity("Parameter id was not a valid integer.")
+
+    command = custom_commands.get_by_id(command_id)
+
+    if command is None:
+        return page_not_found("")
+
+    return jsonify(command)
+
+
+@app.errorhandler(404)
+def page_not_found(e):
+    return jsonify({"error": "No resource could be found matching the given URL."}), 404
+
+
+@app.errorhandler(422)
+def unprocessable_entity(e):
+    return jsonify({"error": e}), 422
+
+
+if __name__ == "__main__":
+    app.run()

cogs/events.py

@@ -5,6 +5,7 @@ from discord.ext import commands
 from functions import checks, easterEggResponses
 from functions.database import stats, muttn, custom_commands, commands as command_stats
 import pytz
+from settings import READY_MESSAGE, SANDBOX, STATUS_MESSAGE
 import time
 import traceback
 
@@ -30,20 +31,13 @@ class Events(commands.Cog):
         """
         Function called when the bot is ready & done leading.
         """
-        # Change status
+        # Set status
-        with open("files/status.txt", "r") as statusFile:
+        await self.client.change_presence(status=discord.Status.online, activity=discord.Game(STATUS_MESSAGE))
-            status = statusFile.readline()
 
-        await self.client.change_presence(status=discord.Status.online, activity=discord.Game(str(status)))
+        print(READY_MESSAGE)
-
-        # Print a message in the terminal to show that he's ready
-        with open("files/readyMessage.txt", "r") as readyFile:
-            readyMessage = readyFile.readline()
-
-        print(readyMessage)
 
         # Add constants to the client as a botvar
-        self.client.constants = constants.Live if "zandbak" not in readyMessage else constants.Zandbak
+        self.client.constants = constants.Live if SANDBOX else constants.Zandbak
 
     @commands.Cog.listener()
     async def on_message(self, message):

cogs/ipc.py

@@ -0,0 +1,26 @@
+from discord.ext import commands, ipc
+
+
+class IPC(commands.Cog):
+    def __init__(self, client):
+        self.client = client
+
+    @ipc.server.route()
+    async def send_dm(self, data):
+        print("got here")
+        user = self.client.get_user(data.user)
+        await user.send(data.message)
+        print("sent")
+        return True
+
+    @ipc.server.route()
+    async def get_bot_latency(self, data):
+        """
+        Get Didier's latency
+        """
+
+        return self.client.latency * 1000
+
+
+def setup(client):
+    client.add_cog(IPC(client))

cogs/tasks.py

@@ -246,7 +246,7 @@ class Tasks(commands.Cog):
         Task that checks for new Ufora announcements every few minutes
         """
         # Don't run this when testing
-        if self.client.user.id == int(constants.coolerDidierId):
+        if self.client.user.id != int(constants.didierId):
             return
 
         # Get new notifications

cogs/translate.py

@@ -15,8 +15,8 @@ class Translate(commands.Cog):
     def cog_check(self, ctx):
         return not self.client.locked
 
-    # @commands.command(name="Translate", aliases=["Tl", "Trans"], usage="[Tekst] [Van]* [Naar]*")
+    @commands.command(name="Translate", aliases=["Tl", "Trans"], usage="[Tekst] [Van]* [Naar]*")
-    # @help.Category(Category.Words)
+    @help.Category(Category.Words)
     async def translate(self, ctx, query=None, to="nl", fr="auto"):
         if query is None:
             return await ctx.send("Controleer je argumenten.")
@@ -39,9 +39,11 @@ class Translate(commands.Cog):
             embed.set_author(name="Didier Translate")
 
             if fr == "auto":
-                language = translation.extra_data["original-language"]
+                language = translation.src
                 embed.add_field(name="Gedetecteerde taal", value=tc(LANGUAGES[language]))
-                embed.add_field(name="Zekerheid", value="{}%".format(translation.extra_data["confidence"] * 100))
+
+                if translation.extra_data["confidence"] is not None:
+                    embed.add_field(name="Zekerheid", value="{}%".format(translation.extra_data["confidence"] * 100))
 
             embed.add_field(name="Origineel ({})".format(translation.src.upper()), value=query, inline=False)
             embed.add_field(name="Vertaling ({})".format(to.upper()), value=translation.text)

didier.py

@@ -1,33 +1,19 @@
 import discord
-from discord.ext import commands
 from dotenv import load_dotenv
 from functions.prefixes import get_prefix
-import os
+from settings import TOKEN
+from startup.didier import Didier
 
+if __name__ == "__main__":
+    load_dotenv(verbose=True)
 
-load_dotenv(verbose=True)
+    # Configure intents (1.5.0)
+    intents = discord.Intents.default()
+    intents.members = True
 
+    client = Didier(command_prefix=get_prefix, case_insensitive=True, intents=intents)
 
-# Configure intents (1.5.0)
+    if client.ipc is not None:
-intents = discord.Intents.default()
+        client.ipc.start()
-intents.members = True
 
-client = commands.Bot(command_prefix=get_prefix, case_insensitive=True, intents=intents)
+    client.run(TOKEN)
-
-# Remove default help because it sucks & I made my own
-client.remove_command("help")
-
-# Load utils first so it can be used in other places & it's not None
-client.load_extension("cogs.utils")
-client.load_extension("cogs.failedchecks")
-client.load_extension("cogs.events")
-
-# Load all remaining cogs
-for file in os.listdir("./cogs"):
-    if file.endswith(".py") and not (file.startswith(("utils", "failedchecks", "events"),)):
-        client.load_extension("cogs.{}".format(file[:-3]))
-
-# Get the token out of the file & run the bot
-with open("files/client.txt", "r") as fp:
-    token = fp.readline()
-client.run(token)

faq.md

@@ -1,5 +1,14 @@
 # FAQ
-Answers to Frequently Asked Questions.
+Answers to Frequently Asked Questions and solutions to issues that may arise.
 
-### Table of Contents
+## Issues installing dotenv
-A list of all questions (in order) so you can easily find what you're looking for.
+
+The name of this package is `python-dotenv`, as listed in `requirements.txt`. The _name_ of the package, however, is just `dotenv`. This confuses PyCharm, which will tell you that you don't have `dotenv` installed as it can't link those two together. You can just click `ignore this requirement` if you don't like the warning.
+
+## Issues installing psycopg2
+
+The `psychopg2` and `psychopg2-binary` packages might cause you some headaches when trying to install them, especially when using PyCharm to install dependencies. The reason for this is because these are `PSQL` packages, and as a result they require you to have `PSQL` installed on your system to function properly.
+
+## Package is not listed in project requirements
+
+This is the exact same case as [Issues installing dotenv](#issues-installing-dotenv), and occurs for packages such as `Quart`. The names of the modules differ from the name used to install it from `pip`. Once again, you can ignore this message.

files/default/hangman.json

@@ -0,0 +1,5 @@
+{
+  "guessed": [],
+  "guesses": 0,
+  "word": ""
+}

files/default/lastTasks.json

@@ -0,0 +1,9 @@
+{
+  "interest": 0,
+  "lost": 0,
+  "poke": 0,
+  "prison": 0,
+  "birthdays": 0,
+  "channels": 0,
+  "remind": 0
+}

files/default/locked.json

@@ -0,0 +1,4 @@
+{
+  "locked": false,
+  "until": -1
+}

files/default/lost.json

@@ -0,0 +1,4 @@
+{
+  "lost": 0,
+  "today": 0
+}

files/default/stats.json

@@ -0,0 +1,19 @@
+{
+  "cf": {
+    "h": 0,
+    "t": 0
+  },
+  "dice": {
+    "2": 0,
+    "5": 0,
+    "3": 0,
+    "6": 0,
+    "1": 0,
+    "4": 0
+  },
+  "rob": {
+    "robs_success": 0,
+    "robs_failed": 0,
+    "bail_paid": 0.0
+  }
+}

files/default/ufora_notifications.json

@@ -0,0 +1,12 @@
+{
+  "Algoritmen en Datastructuren 2": [],
+  "Communicatienetwerken": [],
+  "Computerarchitectuur": [],
+  "Functioneel Programmeren": [],
+  "Multimedia": [],
+  "Software Engineering Lab 1": [],
+  "Statistiek en Probabiliteit": [],
+  "Systeemprogrammeren": [],
+  "Webdevelopment": [],
+  "Wetenschappelijk Rekenen": []
+}

functions/database/custom_commands.py

@@ -127,3 +127,55 @@ def add_alias(command: str, alias: str):
 
     cursor.execute("INSERT INTO custom_command_aliases(command, alias) VALUES(%s, %s)", (command_id, alias,))
     connection.commit()
+
+
+def get_all():
+    """
+    Return a list of all registered custom commands
+    """
+    connection = utils.connect()
+    cursor = connection.cursor()
+
+    cursor.execute("SELECT * FROM custom_commands")
+    commands = cursor.fetchall()
+    ret = []
+
+    # Create a list of all entries
+    for command in commands:
+        dic = {"id": command[0], "name": command[1], "response": command[2]}
+
+        # Find and add aliases
+        cursor.execute("SELECT id, alias FROM custom_command_aliases WHERE command = %s", (command[0],))
+        aliases = cursor.fetchall()
+
+        if aliases:
+            dic["aliases"] = list(map(lambda x: {"id": x[0], "alias": x[1]}, aliases))
+
+        ret.append(dic)
+
+    return ret
+
+
+def get_by_id(command_id: int):
+    """
+    Return a command that matches a given id
+    """
+    connection = utils.connect()
+    cursor = connection.cursor()
+
+    cursor.execute("SELECT * FROM custom_commands WHERE id = %s", (command_id,))
+    command = cursor.fetchone()
+
+    # Nothing found
+    if not command:
+        return None
+
+    dic = {"id": command[0], "name": command[1], "response": command[2]}
+
+    cursor.execute("SELECT id, alias FROM custom_command_aliases WHERE command = %s", (command_id,))
+    aliases = cursor.fetchall()
+
+    if aliases:
+        dic["aliases"] = list(map(lambda x: {"id": x[0], "alias": x[1]}, aliases))
+
+    return dic

functions/database/utils.py

@@ -1,6 +1,5 @@
 import psycopg2
-import json
+from settings import DB_HOST, DB_NAME, DB_USERNAME, DB_PASSWORD
-import os
 
 
 connection = None
@@ -17,15 +16,11 @@ def connect():
 def create_connection():
     global connection
 
-    dir_path = os.path.dirname(os.path.realpath(__file__))
-    with open(os.path.join(dir_path, "../../files/database.json"), "r") as fp:
-        db = json.load(fp)
-
     connection = psycopg2.connect(
-        host=db["host"],
+        host=DB_HOST,
-        database=db["database"],
+        database=DB_NAME,
-        user=db["username"],
+        user=DB_USERNAME,
-        password=db["password"]
+        password=DB_PASSWORD
     )
 
 

ignored.md

@@ -2,28 +2,7 @@
 
 A list of all ignored files with copy-pastable templates. Useful for when you want to work on commands that use these, for obvious reasons. Every file has a copy-pastable template to make it easy for you to use.
 
-These are usually files which would be overkill to make a PSQL table for. Other possibilities are files that are never edited, but should be different on every machine (Discord token, status message, ...).
+These are usually files which would be overkill to make a PSQL table for. Other possibilities are files that are never edited, but should be different on every machine.
-
-### files/client.txt
-
-Contains the application's token to connect to Discord. You can create your own bot & put it's token in this file to run & test Didier code.
-
-    token_goes_here
-
-### files/database.json
-
-Contains the credentials needed to connect to the PSQL database. This is ignored so that I don't have to leak my IP address, but also so that you can set up a local database to mess around without affecting the Live one or having to change any code.
-
-```json
-{
-  "username": "username",
-  "password": "password",
-  "host": "host_address",
-  "database": "database_name"
-}
-```
-
-When connecting to a local PSQL database, `host` should be `"localhost"`.
 
 ### files/hangman.json
 
@@ -64,14 +43,6 @@ Contains a boolean indicating whether or not the server is currently locked, and
 }
 ```
 
-### files/readyMessage.txt
-
-Contains the message printed in your terminal when Didier is ready.
-
-    I'M READY I'M READY I'M READY I'M READY
-    
-In case you were wondering: yes, this is a Spongebob reference.
-
 ### files/stats.json
 
 Contains the stats to track for gambling games. Weren't made as a PSQL table because they would be too long (and every game is different).
@@ -98,12 +69,6 @@ Contains the stats to track for gambling games. Weren't made as a PSQL table bec
 }
 ```
 
-### files/status.txt
-
-Contains Didier's status message for when he logs in. Keep in mind that his activity is set to `Playing `. This was first used in Didier V1 to show whether or not he was in sandbox mode.
-
-    with your Didier Dinks.
-    
 ### files/ufora_notifications.json
 
 Stores ID's of all received Ufora notifications.

readme.md

@@ -51,13 +51,16 @@ When creating a new Didier command, you can add it to a `Category` by adding a d
 
 ```python
 from decorators import help
+from discord.ext import commands
 from enums.help_categories import Category
+from functions import checks
 
 @commands.command(name="Command Name", aliases=["Cn"])
 @commands.check(checks.allowedChannels)
 @help.Category(Category.Currency)
 async def command_name(self, ctx):
     # Command code
+    await ctx.send("Command response")
 ```
 
 This allows commands across multiple Cogs to be classified under the same category in the help page.

requirements.txt

@@ -1,6 +1,8 @@
 python-dotenv==0.14.0
 beautifulsoup4==4.9.1
-discord.py==1.7.0
+discord.py==1.7.3
+git+https://github.com/Rapptz/discord-ext-menus@master
+discord-ext-ipc==2.0.0
 psycopg2==2.8.5
 psycopg2-binary==2.8.5
 python-dateutil==2.6.1
@@ -10,4 +12,6 @@ requests-unixsocket==0.1.5
 tabulate==0.8.7
 yarl==1.4.2
 feedparser==6.0.2
-googletrans==3.0.0
+googletrans==4.0.0rc1
+quart==0.6.15
+Quart-CORS==0.1.3

settings.py

@@ -0,0 +1,33 @@
+from dotenv import load_dotenv
+import os
+
+
+load_dotenv()
+
+
+def _to_bool(value: str) -> bool:
+    """
+    Env variables are strings so this converts them to booleans
+    """
+    return value.lower() in ["true", "1", "y", "yes"]
+
+
+# Sandbox or live
+SANDBOX = _to_bool(os.getenv("SANDBOX", "true"))
+
+# Tokens & API keys
+URBANDICTIONARY = os.getenv("URBANDICTIONARY", "")
+IMGFLIP_NAME = os.getenv("IMGFLIPNAME", "")
+IMGFLIP_PASSWORD = os.getenv("IMGFLIPPASSWORD", "")
+
+# Database credentials
+DB_USERNAME = os.getenv("DBUSERNAME", "")
+DB_PASSWORD = os.getenv("DBPASSWORD", "")
+DB_HOST = os.getenv("DBHOST", "")
+DB_NAME = os.getenv("DBNAME", "")
+
+# Discord-related
+TOKEN = os.getenv("TOKEN", "")
+HOST_IPC = _to_bool(os.getenv("HOSTIPC", "false"))
+READY_MESSAGE = os.getenv("READYMESSAGE", "I'M READY I'M READY I'M READY I'M READY")  # Yes, this is a Spongebob reference
+STATUS_MESSAGE = os.getenv("STATUSMESSAGE", "with your Didier Dinks.")

startup/didier.py

@@ -0,0 +1,46 @@
+from discord.ext import commands, ipc
+from settings import HOST_IPC
+from startup.init_files import check_all
+import os
+
+
+class Didier(commands.Bot):
+    """
+    Main Bot class for Didier
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._host_ipc = HOST_IPC
+
+        # IPC Server
+        # TODO secret key
+        self.ipc = ipc.Server(self, secret_key="SOME_SECRET_KEY") if self._host_ipc else None
+
+        # Cogs that should be loaded before the others
+        self._preload = ("ipc", "utils", "failedchecks", "events",)
+
+        # Remove default help command
+        self.remove_command("help")
+
+        # Load all extensions
+        self.init_extensions()
+
+        # Check missing files
+        check_all()
+
+    def init_extensions(self):
+        # Load initial extensions
+        for ext in self._preload:
+            self.load_extension(f"cogs.{ext}")
+
+        # Load all remaining cogs
+        for file in os.listdir("./cogs"):
+            if file.endswith(".py") and not (file.startswith(self._preload)):
+                self.load_extension("cogs.{}".format(file[:-3]))
+
+    async def on_ipc_ready(self):
+        print("IPC server is ready.")
+
+    async def on_ipc_error(self, endpoint, error):
+        print(endpoint, "raised", error)

startup/init_files.py

@@ -0,0 +1,13 @@
+import json
+from os import path
+
+
+def check_all():
+    files = ["hangman", "lastTasks", "locked", "lost", "stats", "ufora_notifications"]
+
+    for f in files:
+        if not path.isfile(path.join(f"files/{f}.json")):
+            with open(f"files/{f}.json", "w+") as new_file, open(f"files/default/{f}.json", "r") as default:
+                content = json.load(default)
+                json.dump(content, new_file)
+                print(f"Created missing file: files/{f}.json")