From 33d8c301e8df1f61bb1af1a1d9c73a9c5808a6cf Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 25 Feb 2021 10:54:54 +0100 Subject: [PATCH 01/46] Removed shortened stack trace --- app/__main__.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/__main__.py b/app/__main__.py index b4cd3a0..3cf4281 100644 --- a/app/__main__.py +++ b/app/__main__.py @@ -3,14 +3,6 @@ import sys from parser import read_specs_file -# This just displays the error type and message, not the stack trace -def except_hook(ext_type, value, traceback): - sys.stderr.write("{}: {}\n".format(ext_type.__name__, value)) - - -# sys.excepthook = except_hook - - # Define parser parser = argparse.ArgumentParser( description="Backup directories and Docker volumes." From 159d3de72b53026f2bba60b143f1bfbbc73385a8 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Thu, 25 Feb 2021 10:59:59 +0100 Subject: [PATCH 02/46] Added recovery flag --- app/__main__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/__main__.py b/app/__main__.py index 3cf4281..fb38e8d 100644 --- a/app/__main__.py +++ b/app/__main__.py @@ -7,6 +7,7 @@ from parser import read_specs_file parser = argparse.ArgumentParser( description="Backup directories and Docker volumes." ) + parser.add_argument( "-f", "--file", @@ -15,6 +16,7 @@ parser.add_argument( required=True, help="File containing spec definitions.", ) + parser.add_argument( "-j", "--json", @@ -23,6 +25,17 @@ parser.add_argument( default=False, help="Print out the parsed specs as JSON and exit", ) + +parser.add_argument( + "-r", + "--recover", + action="append", + nargs=2, + metavar=("SPEC", "BACKUP"), + dest="recovers", + help="Recover the given spec; requires two arguments", +) + parser.add_argument( "spec", nargs="*", help="The specs to process. Defaults to all." ) From d6531fdde83cc2149850900963ea2a65dd996fb8 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 24 Apr 2021 19:27:33 +0200 Subject: [PATCH 03/46] Switched to setup.py (closes #2) --- .gitignore | 1 + Makefile | 10 ++++++---- requirements-dev.txt | 10 ---------- setup.py | 19 +++++++++++++++++++ requirements.txt => tests/.gitkeep | 0 5 files changed, 26 insertions(+), 14 deletions(-) delete mode 100644 requirements-dev.txt create mode 100644 setup.py rename requirements.txt => tests/.gitkeep (100%) diff --git a/.gitignore b/.gitignore index 4fc828a..c6622bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ __pycache__/ .venv/ backup_tool +*.egg-info/ diff --git a/Makefile b/Makefile index 5f6e639..37a06ea 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,19 @@ # =====CONFIG===== # Devop environment runs in 3.8 -PYTHON=python3.8 +# TODO switch this to python3.6 +PYTHON=python3.9 # =====RECIPES===== -.venv/bin/activate: requirements.txt requirements-dev.txt +.venv/bin/activate: setup.py '$(PYTHON)' -m venv .venv - .venv/bin/pip install -r requirements.txt -r requirements-dev.txt + .venv/bin/pip install -e .[dev] venv: .venv/bin/activate .PHONY: venv format: venv - @ .venv/bin/black app/*.py app/**/*.py + @ .venv/bin/black setup.py app/*.py app/**/*.py .PHONY: format clean: @@ -33,3 +34,4 @@ app: backup_tool install: app cp backup_tool /usr/local/bin +.PHONY: install diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 3eed54f..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Language server -jedi==0.18.0 - -# Linting & Formatting -black==20.8b1 -flake8==3.8.4 - -# Testing -tox==3.21.1 -pytest==6.2.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..8320683 --- /dev/null +++ b/setup.py @@ -0,0 +1,19 @@ +from setuptools import setup + +setup( + name="backup-tool", + version="0.1.0", + author="Jef Roosens", + description="A utility to simply backing up services.", + # TODO add license + packages=["app", "tests"], + extras_require={ + "dev": [ + "jedi==0.18.0", + "black==20.8b1", + "flake8==3.8.4", + "tox==3.21.1", + "pytest==6.2.1", + ] + }, +) diff --git a/requirements.txt b/tests/.gitkeep similarity index 100% rename from requirements.txt rename to tests/.gitkeep From ccb16281fe6e1b9c30c9fa98b2311e83c8c4ee47 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 24 Apr 2021 19:38:03 +0200 Subject: [PATCH 04/46] Added tox config --- .gitignore | 1 + Makefile | 4 ++++ app/__init__.py | 0 setup.py | 1 - tests/__init__.py | 0 tox.ini | 7 +++++++ 6 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 app/__init__.py create mode 100644 tests/__init__.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index c6622bd..665cfc3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__/ .venv/ backup_tool *.egg-info/ +.tox/ diff --git a/Makefile b/Makefile index 37a06ea..5c6d19b 100644 --- a/Makefile +++ b/Makefile @@ -35,3 +35,7 @@ app: backup_tool install: app cp backup_tool /usr/local/bin .PHONY: install + +test: venv tox.ini + @ .venv/bin/tox +.PHONY: test diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/setup.py b/setup.py index 8320683..7fd3d42 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,6 @@ setup( "black==20.8b1", "flake8==3.8.4", "tox==3.21.1", - "pytest==6.2.1", ] }, ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..0df47e0 --- /dev/null +++ b/tox.ini @@ -0,0 +1,7 @@ +[tox] +envlist = py36,py37,py38,py39 + +[testenv] +deps = pytest +commands = + pytest From 156b2441c2b8be9d60e46d9fcc1f19570d50cfd1 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 24 Apr 2021 19:39:49 +0200 Subject: [PATCH 05/46] Added basic woodpecker config --- .woodpecker.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .woodpecker.yml diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..6e194d9 --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,8 @@ +pipeline: + # TODO add config for all other versions + text-3.9-current: + image: python:3.9 + pull: true + commands: + - pip install -e .[dev] + - tox -e py39 From 79f2161b88f8dbdf0a00dfa1b06a3f96a1f70b6b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 24 Apr 2021 22:22:12 +0200 Subject: [PATCH 06/46] Added (hopefully) complete tox config --- .woodpecker.yml | 31 +++++++++++++++++++++++++++++-- Makefile | 4 ++++ tests/test_succeed.py | 2 ++ tox.ini | 9 ++++++++- 4 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 tests/test_succeed.py diff --git a/.woodpecker.yml b/.woodpecker.yml index 6e194d9..1067855 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,8 +1,35 @@ pipeline: # TODO add config for all other versions - text-3.9-current: - image: python:3.9 + test-3.9: + image: python:3.9-alpine pull: true commands: - pip install -e .[dev] - tox -e py39 + + test-3.8: + image: python:3.8-alpine + pull: true + commands: + - pip install -e .[dev] + - tox -e py38 + + test-3.7: + image: python:3.7-alpine + pull: true + commands: + - pip install -e .[dev] + - tox -e py37 + + test-3.6: + image: python:3.6-alpine + pull: true + commands: + - pip install -e .[dev] + - tox -e py36 + + lint: + image: python:3.6-alpine + commands: + - pip install -e .[dev] + - tox -e lint diff --git a/Makefile b/Makefile index 5c6d19b..3bf01a4 100644 --- a/Makefile +++ b/Makefile @@ -39,3 +39,7 @@ install: app test: venv tox.ini @ .venv/bin/tox .PHONY: test + +lint: venv + @ .venv/bin/tox -e lint +.PHONY: lint diff --git a/tests/test_succeed.py b/tests/test_succeed.py new file mode 100644 index 0000000..a5797d6 --- /dev/null +++ b/tests/test_succeed.py @@ -0,0 +1,2 @@ +def test_succeed(): + pass diff --git a/tox.ini b/tox.ini index 0df47e0..3dd1159 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,14 @@ [tox] -envlist = py36,py37,py38,py39 +envlist = py36,py37,py38,py39,lint [testenv] deps = pytest commands = pytest + +[testenv:lint] +basepython = python3.6 +deps = black==20.8b1 + flake8==3.8.4 +commands = black --check setup.py app + flake8 setup.py app From 05974470edc76191061273ab376e692e9e69fecd Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 24 Apr 2021 22:27:03 +0200 Subject: [PATCH 07/46] Made ci tests parallel --- .woodpecker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.woodpecker.yml b/.woodpecker.yml index 1067855..4016d4a 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,6 +1,7 @@ pipeline: # TODO add config for all other versions test-3.9: + group: test image: python:3.9-alpine pull: true commands: @@ -8,6 +9,7 @@ pipeline: - tox -e py39 test-3.8: + group: test image: python:3.8-alpine pull: true commands: @@ -15,6 +17,7 @@ pipeline: - tox -e py38 test-3.7: + group: test image: python:3.7-alpine pull: true commands: @@ -22,6 +25,7 @@ pipeline: - tox -e py37 test-3.6: + group: test image: python:3.6-alpine pull: true commands: From be3ee6ce0404f8e311d3979925c6b8755c85925d Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 24 Apr 2021 22:31:45 +0200 Subject: [PATCH 08/46] Removed unnecessary ci dep --- .woodpecker.yml | 10 +++++----- setup.py | 7 ++++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 4016d4a..3afc452 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -5,7 +5,7 @@ pipeline: image: python:3.9-alpine pull: true commands: - - pip install -e .[dev] + - pip install -e .[ci] - tox -e py39 test-3.8: @@ -13,7 +13,7 @@ pipeline: image: python:3.8-alpine pull: true commands: - - pip install -e .[dev] + - pip install -e .[ci] - tox -e py38 test-3.7: @@ -21,7 +21,7 @@ pipeline: image: python:3.7-alpine pull: true commands: - - pip install -e .[dev] + - pip install -e .[ci] - tox -e py37 test-3.6: @@ -29,11 +29,11 @@ pipeline: image: python:3.6-alpine pull: true commands: - - pip install -e .[dev] + - pip install -e .[ci] - tox -e py36 lint: image: python:3.6-alpine commands: - - pip install -e .[dev] + - pip install -e .[ci] - tox -e lint diff --git a/setup.py b/setup.py index 7fd3d42..f9826d3 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,11 @@ setup( "black==20.8b1", "flake8==3.8.4", "tox==3.21.1", - ] + ], + "ci": [ + "black==20.8b1", + "flake8==3.8.4", + "tox==3.21.1", + ], }, ) From 3e7a2edf13d7452ae365f07af3136a7164f6228a Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 24 Apr 2021 22:34:15 +0200 Subject: [PATCH 09/46] Removed even more deps --- setup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup.py b/setup.py index f9826d3..2cf6eb6 100644 --- a/setup.py +++ b/setup.py @@ -15,8 +15,6 @@ setup( "tox==3.21.1", ], "ci": [ - "black==20.8b1", - "flake8==3.8.4", "tox==3.21.1", ], }, From 7fbec4992e40d9eddccb2e0b197323e1955a176c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 24 Apr 2021 22:56:27 +0200 Subject: [PATCH 10/46] Hopefully fixed ci error --- .woodpecker.yml | 1 + Makefile | 4 ++-- setup.py | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 3afc452..6c393c9 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -35,5 +35,6 @@ pipeline: lint: image: python:3.6-alpine commands: + - apk update && apk add --no-cache build-base - pip install -e .[ci] - tox -e lint diff --git a/Makefile b/Makefile index 3bf01a4..bd7b550 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ # =====CONFIG===== # Devop environment runs in 3.8 # TODO switch this to python3.6 -PYTHON=python3.9 +PYTHON=python3.6 # =====RECIPES===== .venv/bin/activate: setup.py '$(PYTHON)' -m venv .venv - .venv/bin/pip install -e .[dev] + .venv/bin/pip install -e .[dev] -e .[ci] venv: .venv/bin/activate .PHONY: venv diff --git a/setup.py b/setup.py index 2cf6eb6..75c9c5d 100644 --- a/setup.py +++ b/setup.py @@ -8,14 +8,14 @@ setup( # TODO add license packages=["app", "tests"], extras_require={ + # A developer needs both + "ci": [ + "tox==3.21.1", + ], "dev": [ "jedi==0.18.0", "black==20.8b1", "flake8==3.8.4", - "tox==3.21.1", - ], - "ci": [ - "tox==3.21.1", ], }, ) From 2573e67272881fd6b37514b357c68148c92df966 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 24 Apr 2021 23:09:13 +0200 Subject: [PATCH 11/46] Removed dep list from tox.ini --- Makefile | 6 ++++-- setup.py | 7 ++++--- tox.ini | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index bd7b550..622f36c 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ PYTHON=python3.6 # =====RECIPES===== .venv/bin/activate: setup.py '$(PYTHON)' -m venv .venv - .venv/bin/pip install -e .[dev] -e .[ci] + .venv/bin/pip install -e .[ci] -e .[lint] -e .[dev] venv: .venv/bin/activate .PHONY: venv @@ -36,8 +36,10 @@ install: app cp backup_tool /usr/local/bin .PHONY: install +# We can't force the develop to have all the versions locally, so +# the local tests only include python3.6 test: venv tox.ini - @ .venv/bin/tox + @ .venv/bin/tox -e py36 .PHONY: test lint: venv diff --git a/setup.py b/setup.py index 75c9c5d..fb61212 100644 --- a/setup.py +++ b/setup.py @@ -8,14 +8,15 @@ setup( # TODO add license packages=["app", "tests"], extras_require={ - # A developer needs both "ci": [ "tox==3.21.1", ], - "dev": [ - "jedi==0.18.0", + "lint": [ "black==20.8b1", "flake8==3.8.4", ], + "dev": [ + "jedi==0.18.0", + ], }, ) diff --git a/tox.ini b/tox.ini index 3dd1159..d96d690 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,7 @@ commands = [testenv:lint] basepython = python3.6 -deps = black==20.8b1 - flake8==3.8.4 +deps = .[lint] + commands = black --check setup.py app flake8 setup.py app From 8381756693a6df95ed61dd2e701188bacdae400f Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 24 Apr 2021 23:17:03 +0200 Subject: [PATCH 12/46] Added coverage to testing --- .gitignore | 1 + setup.py | 4 ++++ tox.ini | 9 +++++---- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 665cfc3..0156c35 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ __pycache__/ backup_tool *.egg-info/ .tox/ +.coverage diff --git a/setup.py b/setup.py index fb61212..644626a 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,10 @@ setup( "ci": [ "tox==3.21.1", ], + "test": { + "pytest==6.2.3", + "pytest-cov==2.11.1", + }, "lint": [ "black==20.8b1", "flake8==3.8.4", diff --git a/tox.ini b/tox.ini index d96d690..9e1c1b6 100644 --- a/tox.ini +++ b/tox.ini @@ -2,13 +2,14 @@ envlist = py36,py37,py38,py39,lint [testenv] -deps = pytest +deps = .[test] commands = pytest + pytest --cov=app --cov-fail-under=90 tests/ [testenv:lint] basepython = python3.6 deps = .[lint] - -commands = black --check setup.py app - flake8 setup.py app +commands = + black --check setup.py app + flake8 setup.py app From e24e5aa219785f882a8bf2fcab98318405bbff7c Mon Sep 17 00:00:00 2001 From: Jef Roosens Date: Sat, 24 Apr 2021 23:30:46 +0200 Subject: [PATCH 13/46] Added renovate config --- renovate.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..cbafe69 --- /dev/null +++ b/renovate.json @@ -0,0 +1,3 @@ + { + "$schema": "https://docs.renovatebot.com/renovate-schema.json" +} \ No newline at end of file From 16c3edfb175738e73397baf2ee9f121b8c8ec983 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 25 Apr 2021 09:48:35 +0200 Subject: [PATCH 14/46] Switched to setup.cfg for deps --- Makefile | 2 +- setup.cfg | 20 ++++++++++++++++++++ setup.py | 16 ---------------- 3 files changed, 21 insertions(+), 17 deletions(-) create mode 100644 setup.cfg diff --git a/Makefile b/Makefile index 622f36c..1b3dcbc 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ PYTHON=python3.6 # =====RECIPES===== .venv/bin/activate: setup.py '$(PYTHON)' -m venv .venv - .venv/bin/pip install -e .[ci] -e .[lint] -e .[dev] + .venv/bin/pip install -e .[develop] venv: .venv/bin/activate .PHONY: venv diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..1596273 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,20 @@ +[options.extras_require] +# Used to run the tests inside the CICD pipeline +ci = + tox==3.21.1 + +# Used inside Tox for running tests +test = + pytest==6.2.3 + pytest-cov==2.11.1 + +# Used inside tox for linting +lint = + black==20.8b1 + flake8==3.8.4 + +# Required for the developer +develop = + %(ci)s + %(lint)s + jedi==0.18.0 diff --git a/setup.py b/setup.py index 644626a..0b00eae 100644 --- a/setup.py +++ b/setup.py @@ -7,20 +7,4 @@ setup( description="A utility to simply backing up services.", # TODO add license packages=["app", "tests"], - extras_require={ - "ci": [ - "tox==3.21.1", - ], - "test": { - "pytest==6.2.3", - "pytest-cov==2.11.1", - }, - "lint": [ - "black==20.8b1", - "flake8==3.8.4", - ], - "dev": [ - "jedi==0.18.0", - ], - }, ) From 79fb3c9a114b514e28fad6a21dca45315628af40 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 25 Apr 2021 09:59:18 +0200 Subject: [PATCH 15/46] Added flake8 stuff --- .flake8 | 4 ++++ Makefile | 1 + setup.cfg | 5 +++++ 3 files changed, 10 insertions(+) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..bb31e22 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +# vim: ft=cfg +[flake8] +inline-quotes = double +max-complexity = 7 diff --git a/Makefile b/Makefile index 1b3dcbc..b96d269 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ format: venv clean: rm -rf .venv + rm -rf .tox rm backup_tool .PHONY: clean diff --git a/setup.cfg b/setup.cfg index 1596273..ca07553 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,11 @@ test = lint = black==20.8b1 flake8==3.8.4 + flake8-bugbear==20.1.4 + flake8-comprehensions==3.2.3 + flake8-docstrings==1.5.0 + flake8-print==3.1.4 + flake8-quotes==3.2.0 # Required for the developer develop = From 3277af2ac5df2f7a85d8978ed99d2c08f824bb3b Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 25 Apr 2021 18:10:37 +0200 Subject: [PATCH 16/46] Documented skeleton.py --- app/skeleton.py | 55 ++++++++++++++++++++++++++++++++++++++---- app/specs/directory.py | 4 +-- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/app/skeleton.py b/app/skeleton.py index 07c6afb..9fe48a6 100644 --- a/app/skeleton.py +++ b/app/skeleton.py @@ -1,21 +1,55 @@ +"""Handles merging with the skeleton config.""" from typing import Dict class InvalidKeyError(Exception): - def __init__(self, key): + """Thrown when a config file contains an invalid key.""" + + def __init__(self, key: str): + """ + Create a new InvalidKeyError object with the given key. + + Args: + key: the invalid key + """ self.message = "Invalid key: {}".format(key) super().__init__(key) class MissingKeyError(Exception): - def __init__(self, key): + """Thrown when a required key is missing from a config.""" + + def __init__(self, key: str): + """ + Create a new MissingKeyError object with the given key. + + Args: + key: the invalid key + """ self.message = "Missing key: {}".format(key) super().__init__(key) def merge(*dicts: [Dict]) -> Dict: + """ + Merge multiple dicts into one. + + It reads the dicts from left to right, always preferring the "right" + dictionary's values. Therefore, the dictionaries should be sorted from + least important to most important (e.g. a default values skeleton should be + to the left of a dict of selected values). + + Args: + dicts: the dictionaries to merge + + Returns: + a new dictionary representing the merged dictionaries + + Todo: + * Make sure an infinite loop is not possible + """ # Base cases if len(dicts) == 0: return {} @@ -45,10 +79,20 @@ def merge(*dicts: [Dict]) -> Dict: def merge_with_skeleton(data: Dict, skel: Dict) -> Dict: """ - Compare a dict with a given skeleton dict, and fill in default values where - needed. - """ + Merge a dictionary with a skeleton containing default values. + The skeleton not only defines what the default values are, but also + enforces a certain shape. This allows us to define a config file using a + dictionary and parse it. + + Args: + data: dictionary containing the selected config values + skel: dictionary containing the skeleton (aka the def) + + Todo: + * Check if an infinite loop is possible + * Split info less complex functions + """ # First, check for illegal keys for key in data: if key not in skel: @@ -66,6 +110,7 @@ def merge_with_skeleton(data: Dict, skel: Dict) -> Dict: # Error if value is not same type as default value elif type(data[key]) != type(value) and value is not None: + # TODO make this error message more verbose raise TypeError("Invalid value type") # Recurse into dicts diff --git a/app/specs/directory.py b/app/specs/directory.py index 9747b5b..4fa0ffe 100644 --- a/app/specs/directory.py +++ b/app/specs/directory.py @@ -6,9 +6,7 @@ from datetime import datetime class DirectorySpec(Spec): - """ - A spec for backing up a local directory. - """ + """A spec for backing up a local directory.""" _SKEL = { "source": None, From d513a03c4a89173ea5c34478c35bfa9cae11a94e Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 25 Apr 2021 18:27:57 +0200 Subject: [PATCH 17/46] Added parser.py docstrings --- app/parser.py | 12 ++++++++++++ app/skeleton.py | 3 +++ 2 files changed, 15 insertions(+) diff --git a/app/parser.py b/app/parser.py index bbac40c..dcd9fb1 100644 --- a/app/parser.py +++ b/app/parser.py @@ -1,3 +1,4 @@ +"""Handles parsing a config file from disk.""" import yaml from pathlib import Path from typing import List, Union @@ -6,9 +7,19 @@ import skeleton def read_specs_file(path: Union[str, Path]) -> List[Spec]: + """ + Read a config file and merge it with the skeleton. + + Args: + path: path to the yaml config file + + Returns: + A list of specs, parsed from the config. + """ with open(path, "r") as yaml_file: data = yaml.safe_load(yaml_file) + # NOTE shouldn't this be defined as a top-level variable? categories = [ ("directories", DirectorySpec), ("volumes", VolumeSpec), @@ -23,6 +34,7 @@ def read_specs_file(path: Union[str, Path]) -> List[Spec]: # Check what defaults are present defaults = {} + if data.get("defaults"): if data["defaults"].get("all"): defaults = skeleton.merge(defaults, data["defaults"]["all"]) diff --git a/app/skeleton.py b/app/skeleton.py index 9fe48a6..5391e1b 100644 --- a/app/skeleton.py +++ b/app/skeleton.py @@ -89,6 +89,9 @@ def merge_with_skeleton(data: Dict, skel: Dict) -> Dict: data: dictionary containing the selected config values skel: dictionary containing the skeleton (aka the def) + Returns: + a new dictionary representing the two merged dictionaries + Todo: * Check if an infinite loop is possible * Split info less complex functions From ecfa6fe7b7913462adbbacfb26f5bf1009fdeac8 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sun, 25 Apr 2021 19:26:12 +0200 Subject: [PATCH 18/46] Added documentation --- app/__main__.py | 88 +++++++++++++++++++++++------------------- app/specs/__init__.py | 3 ++ app/specs/container.py | 22 +++++++++-- app/specs/spec.py | 14 ++++++- 4 files changed, 83 insertions(+), 44 deletions(-) diff --git a/app/__main__.py b/app/__main__.py index b4cd3a0..fec66e1 100644 --- a/app/__main__.py +++ b/app/__main__.py @@ -1,3 +1,4 @@ +"""The main entrypoint of the program.""" import argparse import sys from parser import read_specs_file @@ -5,55 +6,62 @@ from parser import read_specs_file # This just displays the error type and message, not the stack trace def except_hook(ext_type, value, traceback): + """ + Make errors not show the stracktrace to stdout. + + Todo: + * Replace this with proper error handling + """ sys.stderr.write("{}: {}\n".format(ext_type.__name__, value)) # sys.excepthook = except_hook -# Define parser -parser = argparse.ArgumentParser( - description="Backup directories and Docker volumes." -) -parser.add_argument( - "-f", - "--file", - action="append", - dest="file", - required=True, - help="File containing spec definitions.", -) -parser.add_argument( - "-j", - "--json", - action="store_const", - const=True, - default=False, - help="Print out the parsed specs as JSON and exit", -) -parser.add_argument( - "spec", nargs="*", help="The specs to process. Defaults to all." -) +if __name__ == "__main__": + # Define parser + parser = argparse.ArgumentParser( + description="Backup directories and Docker volumes." + ) + parser.add_argument( + "-f", + "--file", + action="append", + dest="file", + required=True, + help="File containing spec definitions.", + ) + parser.add_argument( + "-j", + "--json", + action="store_const", + const=True, + default=False, + help="Print out the parsed specs as JSON and exit", + ) + parser.add_argument( + "spec", nargs="*", help="The specs to process. Defaults to all." + ) -# Parse arguments -args = parser.parse_args() -specs = sum([read_specs_file(path) for path in args.file], []) + # Parse arguments + args = parser.parse_args() + specs = sum((read_specs_file(path) for path in args.file), []) -# Filter specs if needed -if args.spec: - specs = list(filter(lambda s: s.name in args.spec, specs)) + # Filter specs if needed + if args.spec: + specs = list(filter(lambda s: s.name in args.spec, specs)) -# Dump parsed data as json -if args.json: - import json + # Dump parsed data as json + if args.json: + import json - print(json.dumps([spec.to_dict() for spec in specs], indent=4)) + # TODO replace this with error handling system + print(json.dumps([spec.to_dict() for spec in specs], indent=4)) -else: - # Run the backups - if not specs: - print("No specs, exiting.") - sys.exit(0) + elif not specs: + # TODO replace this with error handling system + print("No specs, exiting.") + sys.exit(0) - for spec in specs: - spec.backup() + for spec in specs: + spec.backup() diff --git a/app/specs/__init__.py b/app/specs/__init__.py index 0192b9e..dd08c1c 100644 --- a/app/specs/__init__.py +++ b/app/specs/__init__.py @@ -1,4 +1,7 @@ +"""Parent module for the various spec types.""" from .spec import Spec from .directory import DirectorySpec from .volume import VolumeSpec from .container import ContainerSpec + +__all__ = ["Spec", "DirectorySpec", "VolumeSpec", "ContainerSpec"] diff --git a/app/specs/container.py b/app/specs/container.py index bef6f8c..76579f0 100644 --- a/app/specs/container.py +++ b/app/specs/container.py @@ -1,3 +1,4 @@ +"""Module defining a Container-based spec.""" from .spec import Spec from typing import Union from pathlib import Path @@ -6,11 +7,10 @@ import subprocess class ContainerSpec(Spec): - """ - A spec for backing up via a container. - """ + """Spec for backing up via a container.""" _SKEL = {"container": None, "command": None, "mountpoint": "/from"} + """The skeleton for the ContainerSpec config.""" def __init__( self, @@ -23,6 +23,22 @@ class ContainerSpec(Spec): mountpoint: str, notify=None, ): + """ + Create a new ContainerSpec object. + + Args: + name: name of the spec (used as an identifier) + container: the Docker container to back up + destination: where to store the backups (gets created if + non-existent) + limit: max amount of backups to keep + command: command to run inside the container. This command should + perform a specified backup and output this data to stdout. This + output then gets piped to a backup file. + extension: the extension of the backup files. + mountpoint: + notify: notifier object (may be None) + """ super().__init__(name, destination, limit, extension, notify) self.container = container diff --git a/app/specs/spec.py b/app/specs/spec.py index 9b372ef..2669ccb 100644 --- a/app/specs/spec.py +++ b/app/specs/spec.py @@ -60,7 +60,19 @@ class Spec: self.extension = extension @classmethod - def skeleton(cls): + def skeleton(cls: "Spec") -> Dict: + """ + Return the skeleton for the given class. + + It works by inspecting the inheritance tree and merging the skeleton + for each of the parents. + + Args: + cls: the class to get the skeleton for + + Returns: + a dictionary containing the skeleton + """ return skeleton.merge( *[val._SKEL for val in reversed(inspect.getmro(cls)[:-1])] ) From 83ea06f0163559e4b1bdf96e95e3849095944a4a Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 26 Apr 2021 17:45:19 +0200 Subject: [PATCH 19/46] Added even more docstrings everywhere --- .flake8 | 6 +++++- app/__init__.py | 1 + app/__main__.py | 12 ++++++------ app/notifier.py | 27 ++++++++++++++++++++++++--- app/parser.py | 5 +++-- app/skeleton.py | 12 ++++-------- app/specs/container.py | 3 ++- app/specs/directory.py | 16 ++++++++++++++++ app/specs/spec.py | 39 +++++++++++++++++++++++++-------------- app/specs/volume.py | 4 +--- 10 files changed, 87 insertions(+), 38 deletions(-) diff --git a/.flake8 b/.flake8 index bb31e22..1d1c7f5 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,8 @@ # vim: ft=cfg [flake8] -inline-quotes = double max-complexity = 7 +docstring-convention=google + +# Required to be compatible with black +extend-ignore = E203,W503 +inline-quotes = double diff --git a/app/__init__.py b/app/__init__.py index e69de29..5927fab 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Module containing all app code.""" diff --git a/app/__main__.py b/app/__main__.py index fec66e1..aaf2942 100644 --- a/app/__main__.py +++ b/app/__main__.py @@ -1,13 +1,13 @@ """The main entrypoint of the program.""" + + import argparse import sys from parser import read_specs_file -# This just displays the error type and message, not the stack trace def except_hook(ext_type, value, traceback): - """ - Make errors not show the stracktrace to stdout. + """Make errors not show the stracktrace to stdout. Todo: * Replace this with proper error handling @@ -59,9 +59,9 @@ if __name__ == "__main__": print(json.dumps([spec.to_dict() for spec in specs], indent=4)) elif not specs: - # TODO replace this with error handling system - print("No specs, exiting.") - sys.exit(0) + # TODO replace this with error handling system + print("No specs, exiting.") + sys.exit(0) for spec in specs: spec.backup() diff --git a/app/notifier.py b/app/notifier.py index 1848775..99b3ffa 100644 --- a/app/notifier.py +++ b/app/notifier.py @@ -1,9 +1,14 @@ +"""Module handling IFTTT notifications.""" + + from typing import List import os import requests class Notifier: + """A notifier object that can send IFTTT notifications.""" + # (positive, negative) _EVENTS = { "backup": ( @@ -15,24 +20,40 @@ class Notifier: "Couldn't restore {name}.", ), } + """The message content for a given event.""" # Placeholder def __init__( self, title: str, events: List[str], endpoint: str, api_key: str = None ): + """Initialize a new Notifier object. + + Args: + title: the notification title to use + events: the event types that should trigger a notification (should + be one of the keys in _EVENTS). + endpoint: IFTTT endpoint name + api_key: your IFTTT API key. If not provided, it will be read from + the IFTTT_API_KEY environment variable. + + Todo: + * Read the API key on init + """ self.title = title self.events = events self.endpoint = endpoint self.api_key = api_key def notify(self, category: str, name: str, status_code: int): - """ + """Send an IFTTT notification. + Args: - category: type of notify (e.g. backup or restore) + category: type of notify (should be one of the keys in _EVENTS). + Only if the category was passed during initialization will the + notification be sent. name: name of the spec status_code: exit code of the command """ - event = "{}_{}".format( category, "success" if status_code == 0 else "failure" ) diff --git a/app/parser.py b/app/parser.py index dcd9fb1..ce5fdf5 100644 --- a/app/parser.py +++ b/app/parser.py @@ -1,4 +1,6 @@ """Handles parsing a config file from disk.""" + + import yaml from pathlib import Path from typing import List, Union @@ -7,8 +9,7 @@ import skeleton def read_specs_file(path: Union[str, Path]) -> List[Spec]: - """ - Read a config file and merge it with the skeleton. + """Read a config file and merge it with the skeleton. Args: path: path to the yaml config file diff --git a/app/skeleton.py b/app/skeleton.py index 5391e1b..b1aabd3 100644 --- a/app/skeleton.py +++ b/app/skeleton.py @@ -6,8 +6,7 @@ class InvalidKeyError(Exception): """Thrown when a config file contains an invalid key.""" def __init__(self, key: str): - """ - Create a new InvalidKeyError object with the given key. + """Create a new InvalidKeyError object with the given key. Args: key: the invalid key @@ -21,8 +20,7 @@ class MissingKeyError(Exception): """Thrown when a required key is missing from a config.""" def __init__(self, key: str): - """ - Create a new MissingKeyError object with the given key. + """Create a new MissingKeyError object with the given key. Args: key: the invalid key @@ -33,8 +31,7 @@ class MissingKeyError(Exception): def merge(*dicts: [Dict]) -> Dict: - """ - Merge multiple dicts into one. + """Merge multiple dicts into one. It reads the dicts from left to right, always preferring the "right" dictionary's values. Therefore, the dictionaries should be sorted from @@ -78,8 +75,7 @@ def merge(*dicts: [Dict]) -> Dict: def merge_with_skeleton(data: Dict, skel: Dict) -> Dict: - """ - Merge a dictionary with a skeleton containing default values. + """Merge a dictionary with a skeleton containing default values. The skeleton not only defines what the default values are, but also enforces a certain shape. This allows us to define a config file using a diff --git a/app/specs/container.py b/app/specs/container.py index 76579f0..7bc2a95 100644 --- a/app/specs/container.py +++ b/app/specs/container.py @@ -1,4 +1,4 @@ -"""Module defining a Container-based spec.""" +"""Module defining a container-based spec.""" from .spec import Spec from typing import Union from pathlib import Path @@ -46,6 +46,7 @@ class ContainerSpec(Spec): self.command = command def backup(self): + """Create a new backup.""" # Remove excess backups self.remove_backups() diff --git a/app/specs/directory.py b/app/specs/directory.py index 4fa0ffe..17048a7 100644 --- a/app/specs/directory.py +++ b/app/specs/directory.py @@ -1,3 +1,6 @@ +"""Module defining a directory-based spec.""" + + from .spec import Spec from pathlib import Path from typing import Union @@ -23,6 +26,18 @@ class DirectorySpec(Spec): extension: str, notify=None, ): + """ + Initialize a new DirectorySpec object. + + Args: + name: name of the spec + source: what directory to back up + destination: where to store the backup + limit: how many backups to keep + command: what command to use to create the backup + extension: extension of the backup files + notify: a Notifier object that handles sending notifications + """ super().__init__(name, destination, limit, extension, notify) self.source = source if type(source) == Path else Path(source) @@ -36,6 +51,7 @@ class DirectorySpec(Spec): self.command = command def backup(self): + """Create a new backup.""" # Remove excess backups self.remove_backups() diff --git a/app/specs/spec.py b/app/specs/spec.py index 2669ccb..d5e0b26 100644 --- a/app/specs/spec.py +++ b/app/specs/spec.py @@ -1,3 +1,6 @@ +"""This module contains the base Spec class.""" + + from pathlib import Path from typing import Union, Dict import skeleton @@ -7,9 +10,7 @@ import inspect class Spec: - """ - Base class for all other spec types. - """ + """Base class for all other spec types.""" _SKEL = { "destination": None, @@ -31,23 +32,25 @@ class Spec: extension: str, notify=None, ): - """ + """Initialize a new Spec object. + + This initializer usually gets called by a subclass's init instead of + directly. + Args: name: name of the spec destination: directory where the backups shall reside limit: max amount of backups notifier: notifier object """ - self.name = name - self.destination = ( - destination if type(destination) == Path else Path(destination) - ) + self.destination = Path(destination) # Create destination if non-existent try: self.destination.mkdir(parents=True, exist_ok=True) + # TODO just make this some checks in advance except FileExistsError: raise NotADirectoryError( "{} already exists, but isn't a directory.".format( @@ -61,8 +64,7 @@ class Spec: @classmethod def skeleton(cls: "Spec") -> Dict: - """ - Return the skeleton for the given class. + """Return the skeleton for the given class. It works by inspecting the inheritance tree and merging the skeleton for each of the parents. @@ -78,10 +80,7 @@ class Spec: ) def remove_backups(self): - """ - Remove all backups exceeding the limit - """ - + """Remove all backups exceeding the limit.""" files = sorted( self.destination.glob("*." + self.extension), key=os.path.getmtime, @@ -93,13 +92,22 @@ class Spec: path.unlink() def backup(self): + """Create a new backup. + + This function should be implemented by the subclasses. + """ raise NotImplementedError() def restore(self): + """Restore a given backup (NOT IMPLEMENTED). + + This function should be implemented by the subclasses. + """ raise NotImplementedError() @classmethod def from_dict(cls, name, obj: Dict, defaults: Dict) -> "Spec": + """Create the class given a dictionary (e.g. from a config).""" # Combine defaults with skeleton, creating new skeleton skel = skeleton.merge(cls.skeleton(), defaults) @@ -109,4 +117,7 @@ class Spec: return cls(name, **obj) def to_dict(self): + """Export the class as a dictionary. + + This function should be imnplemented by the subclasses.""" raise NotImplementedError() diff --git a/app/specs/volume.py b/app/specs/volume.py index cd9d937..f6f0980 100644 --- a/app/specs/volume.py +++ b/app/specs/volume.py @@ -6,9 +6,7 @@ import subprocess class VolumeSpec(Spec): - """ - A spec for backing up a Docker volume. - """ + """A spec for backing up a Docker volume.""" _SKEL = { "volume": None, From 8a7f47dfcdc2d5027a2b0b48029f00aa7743f1a7 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 26 Apr 2021 18:00:56 +0200 Subject: [PATCH 20/46] Three lint errors remain. --- app/specs/container.py | 5 ++--- app/specs/directory.py | 3 +-- app/specs/spec.py | 6 ++++-- app/specs/volume.py | 20 +++++++++++++++++++- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/app/specs/container.py b/app/specs/container.py index 7bc2a95..79d4d3a 100644 --- a/app/specs/container.py +++ b/app/specs/container.py @@ -23,8 +23,7 @@ class ContainerSpec(Spec): mountpoint: str, notify=None, ): - """ - Create a new ContainerSpec object. + """Create a new ContainerSpec object. Args: name: name of the spec (used as an identifier) @@ -36,7 +35,7 @@ class ContainerSpec(Spec): perform a specified backup and output this data to stdout. This output then gets piped to a backup file. extension: the extension of the backup files. - mountpoint: + mountpoint: I don't actually know, this never gets used notify: notifier object (may be None) """ super().__init__(name, destination, limit, extension, notify) diff --git a/app/specs/directory.py b/app/specs/directory.py index 17048a7..34a92e8 100644 --- a/app/specs/directory.py +++ b/app/specs/directory.py @@ -26,8 +26,7 @@ class DirectorySpec(Spec): extension: str, notify=None, ): - """ - Initialize a new DirectorySpec object. + """Initialize a new DirectorySpec object. Args: name: name of the spec diff --git a/app/specs/spec.py b/app/specs/spec.py index d5e0b26..8ba1e28 100644 --- a/app/specs/spec.py +++ b/app/specs/spec.py @@ -41,7 +41,8 @@ class Spec: name: name of the spec destination: directory where the backups shall reside limit: max amount of backups - notifier: notifier object + extension: file extension of the backup files + notify: notifier object to send IFTT notifications """ self.name = name self.destination = Path(destination) @@ -119,5 +120,6 @@ class Spec: def to_dict(self): """Export the class as a dictionary. - This function should be imnplemented by the subclasses.""" + This function should be imnplemented by the subclasses. + """ raise NotImplementedError() diff --git a/app/specs/volume.py b/app/specs/volume.py index f6f0980..27d6e7c 100644 --- a/app/specs/volume.py +++ b/app/specs/volume.py @@ -1,3 +1,6 @@ +"""Module defining a Docker volume-based spec.""" + + from .spec import Spec from typing import Union from pathlib import Path @@ -25,6 +28,18 @@ class VolumeSpec(Spec): extension: str, notify=None, ): + """Initialize a new VolumeSpec object. + + Args: + name: name of the spec + volume: Docker volume to back up + image: base image to use to run backup command + destination: where to store the backup files + limit: max backup files to keep + command: backup command to run within the base image + extension: file extension of the backup files + notify: Notifier object + """ super().__init__(name, destination, limit, extension, notify) self.volume = volume @@ -32,6 +47,7 @@ class VolumeSpec(Spec): self.command = command def backup(self): + """Create a new backup.""" # Remove excess backups self.remove_backups() @@ -40,8 +56,10 @@ class VolumeSpec(Spec): datetime.now().strftime("%Y-%m-%d_%H-%M-%S"), self.extension ) + base_cmd = "docker run --rm -v '{}:/from' -v '{}:/to' -w /from '{}' {}" + process = subprocess.run( - "docker run --rm -v '{}:/from' -v '{}:/to' -w /from '{}' {}".format( + base_cmd.format( self.volume, self.destination, self.image, From 2d06c7feeb276054bdbb85c14374fe1688b82a6c Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 26 Apr 2021 18:14:04 +0200 Subject: [PATCH 21/46] Made merge_with_skeleton "less complex" --- app/skeleton.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/app/skeleton.py b/app/skeleton.py index b1aabd3..b6e871a 100644 --- a/app/skeleton.py +++ b/app/skeleton.py @@ -93,9 +93,10 @@ def merge_with_skeleton(data: Dict, skel: Dict) -> Dict: * Split info less complex functions """ # First, check for illegal keys - for key in data: - if key not in skel: - raise InvalidKeyError(key) + key = next(key not in skel for key in data) + + if key: + raise InvalidKeyError(key) # Then, check the default values for key, value in skel.items(): From c436e12b468a72e343ffc4235e8efcaccacb93c7 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 26 Apr 2021 19:57:54 +0200 Subject: [PATCH 22/46] Added tests for skeleton module --- app/__main__.py | 2 -- app/notifier.py | 2 -- app/parser.py | 2 -- app/specs/directory.py | 2 -- app/specs/spec.py | 3 -- app/specs/volume.py | 2 -- tests/.gitkeep | 0 tests/test_skeleton.py | 69 ++++++++++++++++++++++++++++++++++++++++++ tests/test_succeed.py | 2 -- 9 files changed, 69 insertions(+), 15 deletions(-) delete mode 100644 tests/.gitkeep create mode 100644 tests/test_skeleton.py delete mode 100644 tests/test_succeed.py diff --git a/app/__main__.py b/app/__main__.py index aaf2942..1a674ad 100644 --- a/app/__main__.py +++ b/app/__main__.py @@ -1,6 +1,4 @@ """The main entrypoint of the program.""" - - import argparse import sys from parser import read_specs_file diff --git a/app/notifier.py b/app/notifier.py index 99b3ffa..903e791 100644 --- a/app/notifier.py +++ b/app/notifier.py @@ -1,6 +1,4 @@ """Module handling IFTTT notifications.""" - - from typing import List import os import requests diff --git a/app/parser.py b/app/parser.py index ce5fdf5..b02e5a0 100644 --- a/app/parser.py +++ b/app/parser.py @@ -1,6 +1,4 @@ """Handles parsing a config file from disk.""" - - import yaml from pathlib import Path from typing import List, Union diff --git a/app/specs/directory.py b/app/specs/directory.py index 34a92e8..b8fbc32 100644 --- a/app/specs/directory.py +++ b/app/specs/directory.py @@ -1,6 +1,4 @@ """Module defining a directory-based spec.""" - - from .spec import Spec from pathlib import Path from typing import Union diff --git a/app/specs/spec.py b/app/specs/spec.py index 8ba1e28..f26f54b 100644 --- a/app/specs/spec.py +++ b/app/specs/spec.py @@ -1,7 +1,4 @@ """This module contains the base Spec class.""" - - -from pathlib import Path from typing import Union, Dict import skeleton import os diff --git a/app/specs/volume.py b/app/specs/volume.py index 27d6e7c..403c51b 100644 --- a/app/specs/volume.py +++ b/app/specs/volume.py @@ -1,6 +1,4 @@ """Module defining a Docker volume-based spec.""" - - from .spec import Spec from typing import Union from pathlib import Path diff --git a/tests/.gitkeep b/tests/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_skeleton.py b/tests/test_skeleton.py new file mode 100644 index 0000000..2da81bf --- /dev/null +++ b/tests/test_skeleton.py @@ -0,0 +1,69 @@ +"""Tests for the skeleton module.""" +from app.skeleton import merge + + +def test_merge_empty(): + """Test correct response for an empty merge.""" + assert merge() == {} + + +def test_merge_single(): + """Test merge command with a single input.""" + assert merge({}) == {} + + dic = {"test": "value", "test2": "value2"} + + assert merge(dic) == dic + + +def test_merge_double_no_overlap(): + """Test merge command with two non-overlapping inputs.""" + d1 = {"test": "value", "test2": "value2"} + d2 = {"test3": "value3"} + d_res = {"test": "value", "test2": "value2", "test3": "value3"} + + assert merge(d1, d2) == d_res + + +def test_merge_double_overlap(): + """Test merge command with two overlapping inputs.""" + d1 = {"test": "value", "test2": "value2"} + d2 = {"test2": "value3"} + d_res = {"test": "value", "test2": "value3"} + + assert merge(d1, d2) == d_res + + +def test_merge_triple_no_overlap(): + """Test merge command with three non-overlapping inputs. + + This test tells us that the recursion works. + """ + d1 = {"test": "value", "test2": "value2"} + d2 = {"test3": "value3"} + d3 = {"test4": "value4"} + d_res = { + "test": "value", + "test2": "value2", + "test3": "value3", + "test4": "value4", + } + + assert merge(d1, d2, d3) == d_res + + +def test_merge_triple_overlap(): + """Test merge command with three overlapping inputs. + + This test tells us that the recursion works. + """ + d1 = {"test": "value", "test2": "value2"} + d2 = {"test3": "value3"} + d3 = {"test2": "value4"} + d_res = { + "test": "value", + "test2": "value4", + "test3": "value3", + } + + assert merge(d1, d2, d3) == d_res diff --git a/tests/test_succeed.py b/tests/test_succeed.py deleted file mode 100644 index a5797d6..0000000 --- a/tests/test_succeed.py +++ /dev/null @@ -1,2 +0,0 @@ -def test_succeed(): - pass From 8520b09c4e2ee8e4715c172cb62f7a05a7f0a4bd Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 26 Apr 2021 21:53:40 +0200 Subject: [PATCH 23/46] Started logger module --- app/logger.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 app/logger.py diff --git a/app/logger.py b/app/logger.py new file mode 100644 index 0000000..81d7cd0 --- /dev/null +++ b/app/logger.py @@ -0,0 +1,88 @@ +"""This module contains the logging module.""" +from typing import Union +from pathlib import Path +from datetime import datetime +import sys + + +class Logger: + """A logger class that logs, ya get the point.""" + + LOG_LEVELS = [ + "debug", + "info", + "warning", + "error", + "critical", + ] + """The log levels' names. + + When used as arguments, the counting starts at 1 + instead of 0. + """ + + def __init__( + self, + log_file: Union[Path, str] = None, + append: bool = True, + stdout: bool = True, + log_level: int = 3, + ): + """Initialize a new Logger object. + + Args: + log_file: path to a log file. If any of the folders within the log + file's path don't exist, they will get created. If no value is + specified, no log file is created. + append: wether or not to append to the existing file or overwrite + it. If False, the original file gets deleted during init. + stdout: wether or not to log to stdout as well + log_level: the minimum level to log + """ + self.log_file = Path(log_file) if log_file else None + self.stdout = stdout + self.log_level = log_level + + # Remove the original log file + if not append: + self.log_file.unlink(missing_ok=True) + + def log(self, level: int, message: str): + """Log a message with a specific level. + + Args: + level: log level (index in the LOG_LEVELS variable) + message: the message to log + """ + if level < self.log_level: + return + + level_name = self.LOG_LEVELS[level - 1].upper() + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + log_message = f"[{timestamp}][{level_name}]{message}\n" + + if self.log_file: + self.log_file.write_text(log_message) + + if self.stdout: + sys.stdout.write(log_message) + + def debug(self, message: str): + """Log a debug message.""" + self.log(1, message) + + def info(self, message: str): + """Log an info message.""" + self.log(2, message) + + def warning(self, message: str): + """Log a warning message.""" + self.log(3, message) + + def error(self, message: str): + """Log an error message.""" + self.log(4, message) + + def critical(self, message: str): + """Log a critical message.""" + self.log(5, message) From 8929d743e9b4cb320cb1f32855ddcb4bbffa0b39 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Mon, 26 Apr 2021 22:34:12 +0200 Subject: [PATCH 24/46] Added custom log function; started tests --- .woodpecker.yml | 1 - app/logger.py | 37 +++++++++++++++++++++++++++++-------- tests/test_logger.py | 23 +++++++++++++++++++++++ 3 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 tests/test_logger.py diff --git a/.woodpecker.yml b/.woodpecker.yml index 6c393c9..623affb 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,5 +1,4 @@ pipeline: - # TODO add config for all other versions test-3.9: group: test image: python:3.9-alpine diff --git a/app/logger.py b/app/logger.py index 81d7cd0..245eeed 100644 --- a/app/logger.py +++ b/app/logger.py @@ -47,6 +47,34 @@ class Logger: if not append: self.log_file.unlink(missing_ok=True) + def custom(self, message: str, header: str = None): + """Log a message given a header and a message. + + If a header is provided (aka truthy), the final form of the messsage + wil be: + + `[YYYY-MM-DD HH:MM:SS][header] message` + + Otherwise, it's just: + + `[YYYY-MM-DD HH:MM:SS] message` + + Args: + message: the message to display + header: the header to add to the message + """ + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + log_message = f"[{timestamp}] {message}\n" + + if header: + log_message = f"[{timestamp}][{header}] {message}\n" + + if self.log_file: + self.log_file.write_text(log_message) + + if self.stdout: + sys.stdout.write(log_message) + def log(self, level: int, message: str): """Log a message with a specific level. @@ -58,14 +86,7 @@ class Logger: return level_name = self.LOG_LEVELS[level - 1].upper() - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - log_message = f"[{timestamp}][{level_name}]{message}\n" - - if self.log_file: - self.log_file.write_text(log_message) - - if self.stdout: - sys.stdout.write(log_message) + self.custom(level_name, message) def debug(self, message: str): """Log a debug message.""" diff --git a/tests/test_logger.py b/tests/test_logger.py new file mode 100644 index 0000000..ff2debd --- /dev/null +++ b/tests/test_logger.py @@ -0,0 +1,23 @@ +"""Tests for the logger module.""" +from app.logger import Logger +from datetime import datetime + + +def test_custom_stdout(capfd): + """Test the custom command.""" + logger = Logger() + + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + logger.custom("a message", header="cewl") + + out, _ = capfd.readouterr() + + assert out == f"[{timestamp}][cewl] a message\n" + + +def test_log_stdout(capfd): + """Test the log command with several levels.""" + + logger = Logger() + + # TODO From 50209a04a09d2e543fc9af915ca8346f0e1a9bdd Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 27 Apr 2021 14:17:35 +0000 Subject: [PATCH 25/46] Update dependency flake8 to v3.9.1 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ca07553..eea689c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,7 @@ test = # Used inside tox for linting lint = black==20.8b1 - flake8==3.8.4 + flake8==3.9.1 flake8-bugbear==20.1.4 flake8-comprehensions==3.2.3 flake8-docstrings==1.5.0 From e6cb5a3c2ceeefad5111551c5d8d488133e97de6 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 27 Apr 2021 14:17:49 +0000 Subject: [PATCH 26/46] Update dependency tox to v3.23.0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ca07553..3542298 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [options.extras_require] # Used to run the tests inside the CICD pipeline ci = - tox==3.21.1 + tox==3.23.0 # Used inside Tox for running tests test = From 93d46c696be790f5564f3bbadc2695fe4052f457 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 27 Apr 2021 14:17:56 +0000 Subject: [PATCH 27/46] Update dependency flake8-print to v4 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index ca07553..8e03699 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ lint = flake8-bugbear==20.1.4 flake8-comprehensions==3.2.3 flake8-docstrings==1.5.0 - flake8-print==3.1.4 + flake8-print==4.0.0 flake8-quotes==3.2.0 # Required for the developer From 9810bc2f803555c872fea1b6f032dc66fd1f7fa0 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 27 Apr 2021 16:32:20 +0200 Subject: [PATCH 28/46] Updated renovate.json to allow auto-merging --- renovate.json | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/renovate.json b/renovate.json index cbafe69..6ab3f03 100644 --- a/renovate.json +++ b/renovate.json @@ -1,3 +1,10 @@ { - "$schema": "https://docs.renovatebot.com/renovate-schema.json" -} \ No newline at end of file + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "packageRules": [ + { + "matchUpdateTypes": ["minor", "patch", "pin", "digest"], + "automerge": true, + "automergeType": "branch" + } + ] +} From 5ddd72b1921c65043d75f06dbb27c64ae3779a89 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Tue, 27 Apr 2021 17:18:51 +0200 Subject: [PATCH 29/46] Fixed renovate.json indentation [CI SKIP] --- renovate.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/renovate.json b/renovate.json index 6ab3f03..dca187a 100644 --- a/renovate.json +++ b/renovate.json @@ -1,10 +1,10 @@ - { - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "packageRules": [ - { - "matchUpdateTypes": ["minor", "patch", "pin", "digest"], - "automerge": true, - "automergeType": "branch" - } - ] +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "packageRules": [ + { + "matchUpdateTypes": ["minor", "patch", "pin", "digest"], + "automerge": true, + "automergeType": "branch" + } + ] } From 2d2341f68cee28a354936a97c20b28791053a914 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 27 Apr 2021 16:00:27 +0000 Subject: [PATCH 30/46] Update dependency flake8-bugbear to v20.11.1 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 79b0e43..55eab0c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,7 +12,7 @@ test = lint = black==20.8b1 flake8==3.9.1 - flake8-bugbear==20.1.4 + flake8-bugbear==20.11.1 flake8-comprehensions==3.2.3 flake8-docstrings==1.5.0 flake8-print==4.0.0 From 4cf8028fbad9c1add2997ef7d30fc6ef13abafd1 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Wed, 28 Apr 2021 14:00:47 +0000 Subject: [PATCH 31/46] Update dependency flake8-comprehensions to v3.4.0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 55eab0c..41e6a35 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ lint = black==20.8b1 flake8==3.9.1 flake8-bugbear==20.11.1 - flake8-comprehensions==3.2.3 + flake8-comprehensions==3.4.0 flake8-docstrings==1.5.0 flake8-print==4.0.0 flake8-quotes==3.2.0 From c844e2fa4afe6b76c1051c0aadc7f7c86834ca71 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 28 Apr 2021 16:09:44 +0200 Subject: [PATCH 32/46] Added PyPy 3.7 tests --- .woodpecker.yml | 8 ++++++++ tox.ini | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 623affb..819ac41 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -23,6 +23,14 @@ pipeline: - pip install -e .[ci] - tox -e py37 + test-3.7-pypy: + group: test + image: pypy:3-7-slim + pull: true + commands: + - pip install -e .[ci] + - tex -e pypy37 + test-3.6: group: test image: python:3.6-alpine diff --git a/tox.ini b/tox.ini index 9e1c1b6..847ba45 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py36,py37,py38,py39,lint +envlist = py36,py37,pypy37,py38,py39,lint [testenv] deps = .[test] From 1453de934694f0ce69f2652224ff2efddc03969d Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 28 Apr 2021 16:12:00 +0200 Subject: [PATCH 33/46] Fixed dumb typo --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 819ac41..972f9e0 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -29,7 +29,7 @@ pipeline: pull: true commands: - pip install -e .[ci] - - tex -e pypy37 + - tox -e pypy37 test-3.6: group: test From d39faee995f4d2369c8228ca3e592e4f572b5964 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Wed, 28 Apr 2021 16:19:53 +0200 Subject: [PATCH 34/46] Cleaned up Makefile [CI SKIP] --- Makefile | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index b96d269..e6e77ec 100644 --- a/Makefile +++ b/Makefile @@ -1,27 +1,28 @@ # =====CONFIG===== -# Devop environment runs in 3.8 -# TODO switch this to python3.6 -PYTHON=python3.6 +PYTHON := python3.6 +VENV := .venv # =====RECIPES===== -.venv/bin/activate: setup.py - '$(PYTHON)' -m venv .venv - .venv/bin/pip install -e .[develop] +# Create the virtual environment +$(VENV)/bin/activate: setup.py + '$(PYTHON)' -m venv '$(VENV)' + '$(VENV)/bin/pip' install -e .[develop] -venv: .venv/bin/activate +venv: $(VENV)/bin/activate .PHONY: venv +# Format the codebase using Black format: venv - @ .venv/bin/black setup.py app/*.py app/**/*.py + @ '$(VENV)/bin/black' setup.py app/*.py app/**/*.py .PHONY: format +# Remove any temporary files clean: - rm -rf .venv - rm -rf .tox - rm backup_tool + @ rm -rf '$(VENV)' .tox backup_tool .PHONY: clean +# Pack the package into a zipfile backup_tool: @ cd app && \ zip -r ../app.zip * \ @@ -33,6 +34,7 @@ backup_tool: app: backup_tool .PHONY: app +# Install the app install: app cp backup_tool /usr/local/bin .PHONY: install @@ -40,9 +42,9 @@ install: app # We can't force the develop to have all the versions locally, so # the local tests only include python3.6 test: venv tox.ini - @ .venv/bin/tox -e py36 + @ '$(VENV)/bin/tox' -e py36 .PHONY: test lint: venv - @ .venv/bin/tox -e lint + @ '$(VENV)/bin/tox' -e lint .PHONY: lint From 1394c646e3cd6cdb95eb44b558d59d41138d2825 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Wed, 28 Apr 2021 19:00:45 +0000 Subject: [PATCH 35/46] Update dependency flake8-docstrings to v1.6.0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 41e6a35..9d1c48e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ lint = flake8==3.9.1 flake8-bugbear==20.11.1 flake8-comprehensions==3.4.0 - flake8-docstrings==1.5.0 + flake8-docstrings==1.6.0 flake8-print==4.0.0 flake8-quotes==3.2.0 From c573920e645331461d8f268957b62c8ce46b4c20 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Thu, 29 Apr 2021 07:00:42 +0000 Subject: [PATCH 36/46] Update dependency flake8-bugbear to v21 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9d1c48e..1e29732 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,7 +12,7 @@ test = lint = black==20.8b1 flake8==3.9.1 - flake8-bugbear==20.11.1 + flake8-bugbear==21.4.3 flake8-comprehensions==3.4.0 flake8-docstrings==1.6.0 flake8-print==4.0.0 From fd8f768d6ae359c42a691ad02a1254a1a3b6e546 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Fri, 7 May 2021 20:02:48 +0000 Subject: [PATCH 37/46] Update dependency pytest to v6.2.4 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1e29732..6ba48a7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ ci = # Used inside Tox for running tests test = - pytest==6.2.3 + pytest==6.2.4 pytest-cov==2.11.1 # Used inside tox for linting From 7633c9d55ad5b21626900d38995c0be9a015996a Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sat, 8 May 2021 20:00:27 +0000 Subject: [PATCH 38/46] Update dependency flake8 to v3.9.2 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1e29732..29d8cc5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,7 @@ test = # Used inside tox for linting lint = black==20.8b1 - flake8==3.9.1 + flake8==3.9.2 flake8-bugbear==21.4.3 flake8-comprehensions==3.4.0 flake8-docstrings==1.6.0 From ad31d9a979acbaeccd3a0200997ba9180706d9b9 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 15 May 2021 13:40:11 +0200 Subject: [PATCH 39/46] Started tests for skeleton module --- Makefile | 2 +- app/skeleton.py | 30 ++++++++------ renovate.json | 9 +--- tests/test_dict_merge.py | 69 +++++++++++++++++++++++++++++++ tests/test_skeleton.py | 88 ++++++++++++---------------------------- 5 files changed, 116 insertions(+), 82 deletions(-) create mode 100644 tests/test_dict_merge.py diff --git a/Makefile b/Makefile index e6e77ec..ab0f16e 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ venv: $(VENV)/bin/activate # Format the codebase using Black format: venv - @ '$(VENV)/bin/black' setup.py app/*.py app/**/*.py + @ '$(VENV)/bin/black' setup.py app .PHONY: format # Remove any temporary files diff --git a/app/skeleton.py b/app/skeleton.py index b6e871a..76d9774 100644 --- a/app/skeleton.py +++ b/app/skeleton.py @@ -1,33 +1,39 @@ """Handles merging with the skeleton config.""" -from typing import Dict +from typing import Dict, Union, List class InvalidKeyError(Exception): """Thrown when a config file contains an invalid key.""" - def __init__(self, key: str): + def __init__(self, keys: Union[str, List[str]]): """Create a new InvalidKeyError object with the given key. Args: - key: the invalid key + keys: the invalid key(s) """ - self.message = "Invalid key: {}".format(key) + if type(keys) == str: + keys = [keys] - super().__init__(key) + self.message = "Invalid key(s): {}".format(", ".join(keys)) + + super().__init__() class MissingKeyError(Exception): """Thrown when a required key is missing from a config.""" - def __init__(self, key: str): + def __init__(self, keys: Union[str, List[str]]): """Create a new MissingKeyError object with the given key. Args: - key: the invalid key + keys: the invalid key(s) """ - self.message = "Missing key: {}".format(key) + if type(keys) == str: + keys = [keys] - super().__init__(key) + self.message = "Missing key(s): {}".format(", ".join(keys)) + + super().__init__() def merge(*dicts: [Dict]) -> Dict: @@ -93,10 +99,10 @@ def merge_with_skeleton(data: Dict, skel: Dict) -> Dict: * Split info less complex functions """ # First, check for illegal keys - key = next(key not in skel for key in data) + invalid_keys = list(filter(lambda k: k not in skel, data)) - if key: - raise InvalidKeyError(key) + if invalid_keys: + raise InvalidKeyError(invalid_keys) # Then, check the default values for key, value in skel.items(): diff --git a/renovate.json b/renovate.json index dca187a..9775346 100644 --- a/renovate.json +++ b/renovate.json @@ -1,10 +1,3 @@ { - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "packageRules": [ - { - "matchUpdateTypes": ["minor", "patch", "pin", "digest"], - "automerge": true, - "automergeType": "branch" - } - ] + "$schema": "https://docs.renovatebot.com/renovate-schema.json" } diff --git a/tests/test_dict_merge.py b/tests/test_dict_merge.py new file mode 100644 index 0000000..2da81bf --- /dev/null +++ b/tests/test_dict_merge.py @@ -0,0 +1,69 @@ +"""Tests for the skeleton module.""" +from app.skeleton import merge + + +def test_merge_empty(): + """Test correct response for an empty merge.""" + assert merge() == {} + + +def test_merge_single(): + """Test merge command with a single input.""" + assert merge({}) == {} + + dic = {"test": "value", "test2": "value2"} + + assert merge(dic) == dic + + +def test_merge_double_no_overlap(): + """Test merge command with two non-overlapping inputs.""" + d1 = {"test": "value", "test2": "value2"} + d2 = {"test3": "value3"} + d_res = {"test": "value", "test2": "value2", "test3": "value3"} + + assert merge(d1, d2) == d_res + + +def test_merge_double_overlap(): + """Test merge command with two overlapping inputs.""" + d1 = {"test": "value", "test2": "value2"} + d2 = {"test2": "value3"} + d_res = {"test": "value", "test2": "value3"} + + assert merge(d1, d2) == d_res + + +def test_merge_triple_no_overlap(): + """Test merge command with three non-overlapping inputs. + + This test tells us that the recursion works. + """ + d1 = {"test": "value", "test2": "value2"} + d2 = {"test3": "value3"} + d3 = {"test4": "value4"} + d_res = { + "test": "value", + "test2": "value2", + "test3": "value3", + "test4": "value4", + } + + assert merge(d1, d2, d3) == d_res + + +def test_merge_triple_overlap(): + """Test merge command with three overlapping inputs. + + This test tells us that the recursion works. + """ + d1 = {"test": "value", "test2": "value2"} + d2 = {"test3": "value3"} + d3 = {"test2": "value4"} + d_res = { + "test": "value", + "test2": "value4", + "test3": "value3", + } + + assert merge(d1, d2, d3) == d_res diff --git a/tests/test_skeleton.py b/tests/test_skeleton.py index 2da81bf..c14fa0c 100644 --- a/tests/test_skeleton.py +++ b/tests/test_skeleton.py @@ -1,69 +1,35 @@ -"""Tests for the skeleton module.""" -from app.skeleton import merge +"""Tests wether the skeleton merge works.""" +from app.skeleton import merge_with_skeleton, MissingKeyError, InvalidKeyError +import pytest -def test_merge_empty(): - """Test correct response for an empty merge.""" - assert merge() == {} - - -def test_merge_single(): - """Test merge command with a single input.""" - assert merge({}) == {} - - dic = {"test": "value", "test2": "value2"} - - assert merge(dic) == dic - - -def test_merge_double_no_overlap(): - """Test merge command with two non-overlapping inputs.""" - d1 = {"test": "value", "test2": "value2"} - d2 = {"test3": "value3"} - d_res = {"test": "value", "test2": "value2", "test3": "value3"} - - assert merge(d1, d2) == d_res - - -def test_merge_double_overlap(): - """Test merge command with two overlapping inputs.""" - d1 = {"test": "value", "test2": "value2"} - d2 = {"test2": "value3"} - d_res = {"test": "value", "test2": "value3"} - - assert merge(d1, d2) == d_res - - -def test_merge_triple_no_overlap(): - """Test merge command with three non-overlapping inputs. - - This test tells us that the recursion works. - """ - d1 = {"test": "value", "test2": "value2"} - d2 = {"test3": "value3"} - d3 = {"test4": "value4"} - d_res = { - "test": "value", - "test2": "value2", - "test3": "value3", - "test4": "value4", +def test_single_invalid_key(): + """Tests wether an InvalidKeyError is correctly thrown for a single key.""" + data = { + "test": 1, + "test2": "test" + } + skel = { + "test": None, } - assert merge(d1, d2, d3) == d_res + with pytest.raises(InvalidKeyError) as e_info: + merge_with_skeleton(data, skel) + + assert e_info.value.message == "Invalid key(s): test2" -def test_merge_triple_overlap(): - """Test merge command with three overlapping inputs. - - This test tells us that the recursion works. - """ - d1 = {"test": "value", "test2": "value2"} - d2 = {"test3": "value3"} - d3 = {"test2": "value4"} - d_res = { - "test": "value", - "test2": "value4", - "test3": "value3", +def test_single_missing_key(): + """Tests wether a MissingKeyError is correctly thrown for a single key.""" + data = { + "test": 1, + } + skel = { + "test": None, + "test2": None, } - assert merge(d1, d2, d3) == d_res + with pytest.raises(MissingKeyError) as e_info: + merge_with_skeleton(data, skel) + + assert e_info.value.message == "Missing key(s): test2" From 0dc1b3ded42b2e1e9282d6f41d6a4d59b34d4389 Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 15 May 2021 13:46:36 +0200 Subject: [PATCH 40/46] Moved exceptions to own file; added some more tests --- app/exceptions.py | 36 ++++++++++++++++++++++++++++++++++++ app/skeleton.py | 37 ++----------------------------------- tests/test_skeleton.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 35 deletions(-) create mode 100644 app/exceptions.py diff --git a/app/exceptions.py b/app/exceptions.py new file mode 100644 index 0000000..1ebfe95 --- /dev/null +++ b/app/exceptions.py @@ -0,0 +1,36 @@ +"""Common exceptions raised by the program.""" +from typing import Union, List + + +class InvalidKeyError(Exception): + """Thrown when a config file contains an invalid key.""" + + def __init__(self, keys: Union[str, List[str]]): + """Create a new InvalidKeyError object with the given key. + + Args: + keys: the invalid key(s) + """ + if type(keys) == str: + keys = [keys] + + self.message = "Invalid key(s): {}".format(", ".join(keys)) + + super().__init__() + + +class MissingKeyError(Exception): + """Thrown when a required key is missing from a config.""" + + def __init__(self, keys: Union[str, List[str]]): + """Create a new MissingKeyError object with the given key. + + Args: + keys: the invalid key(s) + """ + if type(keys) == str: + keys = [keys] + + self.message = "Missing key(s): {}".format(", ".join(keys)) + + super().__init__() diff --git a/app/skeleton.py b/app/skeleton.py index 76d9774..c12bd45 100644 --- a/app/skeleton.py +++ b/app/skeleton.py @@ -1,39 +1,6 @@ """Handles merging with the skeleton config.""" -from typing import Dict, Union, List - - -class InvalidKeyError(Exception): - """Thrown when a config file contains an invalid key.""" - - def __init__(self, keys: Union[str, List[str]]): - """Create a new InvalidKeyError object with the given key. - - Args: - keys: the invalid key(s) - """ - if type(keys) == str: - keys = [keys] - - self.message = "Invalid key(s): {}".format(", ".join(keys)) - - super().__init__() - - -class MissingKeyError(Exception): - """Thrown when a required key is missing from a config.""" - - def __init__(self, keys: Union[str, List[str]]): - """Create a new MissingKeyError object with the given key. - - Args: - keys: the invalid key(s) - """ - if type(keys) == str: - keys = [keys] - - self.message = "Missing key(s): {}".format(", ".join(keys)) - - super().__init__() +from typing import Dict +from .exceptions import InvalidKeyError, MissingKeyError def merge(*dicts: [Dict]) -> Dict: diff --git a/tests/test_skeleton.py b/tests/test_skeleton.py index c14fa0c..6cebba6 100644 --- a/tests/test_skeleton.py +++ b/tests/test_skeleton.py @@ -1,5 +1,6 @@ """Tests wether the skeleton merge works.""" from app.skeleton import merge_with_skeleton, MissingKeyError, InvalidKeyError +from app.exceptions import InvalidKeyError, MissingKeyError import pytest @@ -19,6 +20,23 @@ def test_single_invalid_key(): assert e_info.value.message == "Invalid key(s): test2" +def test_multiple_invalid_keys(): + """Tests wether an InvalidKeyError is thrown for multiple keys.""" + data = { + "test": 1, + "test2": "test", + "test3": "test", + } + skel = { + "test": None, + } + + with pytest.raises(InvalidKeyError) as e_info: + merge_with_skeleton(data, skel) + + assert e_info.value.message == "Invalid key(s): test2, test3" + + def test_single_missing_key(): """Tests wether a MissingKeyError is correctly thrown for a single key.""" data = { @@ -33,3 +51,20 @@ def test_single_missing_key(): merge_with_skeleton(data, skel) assert e_info.value.message == "Missing key(s): test2" + + +def test_multiple_missing_keys(): + """Tests wether a MissingKeyError is correctly thrown for multiple keys.""" + data = { + "test": 1, + } + skel = { + "test": None, + "test2": None, + "test3": None, + } + + with pytest.raises(MissingKeyError) as e_info: + merge_with_skeleton(data, skel) + + assert e_info.value.message == "Missing key(s): test2, test3" From f605c9738ff5614849598fb65f370ccb7a5a7796 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sat, 15 May 2021 12:00:23 +0000 Subject: [PATCH 41/46] Update dependency tox to v3.23.1 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 1e29732..617b4e6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [options.extras_require] # Used to run the tests inside the CICD pipeline ci = - tox==3.23.0 + tox==3.23.1 # Used inside Tox for running tests test = From 8734cdf180799fa14f2d8ca646919b56d90bfcad Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 15 May 2021 14:04:34 +0200 Subject: [PATCH 42/46] Added new exception [CI SKIP] --- app/exceptions.py | 19 +++++++++++++++++++ tests/test_skeleton.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/app/exceptions.py b/app/exceptions.py index 1ebfe95..b82bad8 100644 --- a/app/exceptions.py +++ b/app/exceptions.py @@ -34,3 +34,22 @@ class MissingKeyError(Exception): self.message = "Missing key(s): {}".format(", ".join(keys)) super().__init__() + + +class InvalidValueError(Exception): + """Thrown when a key contains an invalid value.""" + + def __init__(self, key: str, expected: str, actual: str): + """Create a new InvalidValueError given the arguments. + + Args: + key: the key containing the invalid value + expected: name of the expected type + actual: name of the actual type + """ + self.message = ( + f"Invalid value for key {key}: expected {expected}, " + f"got {actual}" + ) + + super().__init__() diff --git a/tests/test_skeleton.py b/tests/test_skeleton.py index 6cebba6..9ade87e 100644 --- a/tests/test_skeleton.py +++ b/tests/test_skeleton.py @@ -1,5 +1,5 @@ """Tests wether the skeleton merge works.""" -from app.skeleton import merge_with_skeleton, MissingKeyError, InvalidKeyError +from app.skeleton import merge_with_skeleton from app.exceptions import InvalidKeyError, MissingKeyError import pytest From 1a900056cf3b5cc53b4c20d9ddc7e7fffd83180a Mon Sep 17 00:00:00 2001 From: Chewing_Bever Date: Sat, 15 May 2021 16:21:48 +0200 Subject: [PATCH 43/46] Switched CI to non-tox matrix build --- .woodpecker.yml | 50 +++++++++++++++++-------------------------------- 1 file changed, 17 insertions(+), 33 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 972f9e0..9673933 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,47 +1,31 @@ +matrix: + PYTHON_VERSION: + - 3.6 + - 3.7 + - 3.8 + - 3.9 + pipeline: - test-3.9: + test: group: test - image: python:3.9-alpine + image: python:${PYTHON_VERSION}-alpine pull: true commands: - - pip install -e .[ci] - - tox -e py39 + - pip install -e .[test] + - pytest --cov=app --cov-fail-under=90 tests/ - test-3.8: - group: test - image: python:3.8-alpine - pull: true - commands: - - pip install -e .[ci] - - tox -e py38 - - test-3.7: - group: test - image: python:3.7-alpine - pull: true - commands: - - pip install -e .[ci] - - tox -e py37 - - test-3.7-pypy: + test-pypy: group: test image: pypy:3-7-slim pull: true commands: - - pip install -e .[ci] - - tox -e pypy37 - - test-3.6: - group: test - image: python:3.6-alpine - pull: true - commands: - - pip install -e .[ci] - - tox -e py36 + - pip install -e .[test] + - pytest --cov=app --cov-fail-under=90 tests/ lint: image: python:3.6-alpine commands: - apk update && apk add --no-cache build-base - - pip install -e .[ci] - - tox -e lint + - pip install -e .[lint] + - black --check setup.py app + - flake8 setup.py app From df9147b7319280c9ba88b7ab89be04b6ce2a7db2 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sat, 15 May 2021 15:01:49 +0000 Subject: [PATCH 44/46] Update dependency flake8-comprehensions to v3.5.0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 74cd3ad..fe5d279 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ lint = black==20.8b1 flake8==3.9.2 flake8-bugbear==21.4.3 - flake8-comprehensions==3.4.0 + flake8-comprehensions==3.5.0 flake8-docstrings==1.6.0 flake8-print==4.0.0 flake8-quotes==3.2.0 From 15b90ff67037e7484d8874dcd8e7fd59dfef90c3 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sat, 15 May 2021 15:01:53 +0000 Subject: [PATCH 45/46] Update dependency pytest-cov to v2.12.0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 74cd3ad..7c5f8fa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,7 @@ ci = # Used inside Tox for running tests test = pytest==6.2.4 - pytest-cov==2.11.1 + pytest-cov==2.12.0 # Used inside tox for linting lint = From 92504d28843ab10a55f91de73c0f5b881e104c5c Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 1 Jun 2021 18:01:49 +0000 Subject: [PATCH 46/46] Update dependency pytest-cov to v2.12.1 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 4dd01c8..568b586 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,7 @@ ci = # Used inside Tox for running tests test = pytest==6.2.4 - pytest-cov==2.12.0 + pytest-cov==2.12.1 # Used inside tox for linting lint =