diff --git a/backups/.gitignore b/backups/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/backups/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/backups/README.md b/backups/README.md new file mode 100644 index 0000000..595073f --- /dev/null +++ b/backups/README.md @@ -0,0 +1,4 @@ +# Backups +I wrote this Python program to manage backups of the stuff running on our +server. I know there's probably better ways to do this, but I really liked +working on this and it works good enough for our usecase. diff --git a/backups/app/__main__.py b/backups/app/__main__.py new file mode 100644 index 0000000..c340221 --- /dev/null +++ b/backups/app/__main__.py @@ -0,0 +1,42 @@ +import argparse +import sys +from specs import parse_specs_file + + +# This just displays the error type and message, not the stack trace +def except_hook(ext_type, value, traceback): + sys.stderr.write("{}: {}\n".format(ext_type.__name__, value)) + +sys.excepthook = except_hook + + +# Define parser +parser = argparse.ArgumentParser( + description='Backup directories and Docker volumes.') +parser.add_argument('-s', '--spec', action='append', dest='specs', + help='Spec to back up. If not specified, all specs in ' + 'in provided files are processed.') +parser.add_argument('-j', '--json', action='store_const', const=True, + default=False, help='Print out the parsed specs as JSON ' + 'and exit') +parser.add_argument('file', nargs='+', + help='A YAML file containg specs.') + +# Parse arguments +args = parser.parse_args() +specs = sum([parse_specs_file(path) for path in args.file], []) + +# Filter specs if needed +if args.specs: + specs = filter(lambda s: s.name in args.specs, specs) + +# Dump parsed data as json +if args.json: + import json + print(json.dumps([spec.to_dict() for spec in specs], indent=4)) + +else: + pass + # Run the backups + # for spec in specs: + # spec.backup() diff --git a/backups/app/notifier.py b/backups/app/notifier.py new file mode 100644 index 0000000..e69de29 diff --git a/backups/app/specs/__init__.py b/backups/app/specs/__init__.py new file mode 100644 index 0000000..5104164 --- /dev/null +++ b/backups/app/specs/__init__.py @@ -0,0 +1,2 @@ +from .specs import Spec +from .parser import parse_specs_file diff --git a/backups/app/specs/parser.py b/backups/app/specs/parser.py new file mode 100644 index 0000000..d490721 --- /dev/null +++ b/backups/app/specs/parser.py @@ -0,0 +1,112 @@ +import yaml +from pathlib import Path +from specs import Spec +from typing import List, Dict + + +class InvalidKeyError(Exception): + def __init__(self, key): + message = "Invalid key: {}".format(key) + + super().__init__(key) + + +class MissingKeyError(Exception): + def __init__(self, key): + message = "Missing key: {}".format(key) + + super().__init__(key) + + +def parse_specs_file(path: Path) -> List[Spec]: + """ + Parse a YAML file defining backup specs. + + Args: + path: path to the specs file + + Returns: + A list of specs + """ + + # Skeleton of a spec config + # If a value is None, this means it doesn't have a default value and must be + # defined + spec_skel = { + "source": None, + "destination": None, + "limit": None, + "volume": False, + "notify": { + "title": "Backup Notification", + "events": ["success"] + } + } + + # Read YAML file + with open(path, "r") as yaml_file: + data = yaml.load(yaml_file, Loader=yaml.Loader) + + # Check specs section exists + if "specs" not in data: + raise MissingKeyError("specs") + + # TODO check if only specs section exists + + specs = [] + # Check format for each spec + for key in data["specs"]: + specs.append(Spec.from_dict(key, combine_with_skeleton( + data["specs"][key], spec_skel) + )) + + return specs + + +def combine_with_skeleton(data: Dict, skel: Dict) -> Dict: + """ + Compare a dict with a given skeleton dict, and fill in default values where + needed. + """ + + # First, check for illegal keys + for key in data: + if key not in skel: + raise InvalidKeyError(key) + + # Then, check the default values + for key, value in skel.items(): + if key not in data: + # Raise error if there's not default value + if value is None: + raise MissingKeyError(key) + + # Replace with default value + data[key] = value + + # Error if value is not same type as default value + elif type(data[key]) != type(value) and value is not None: + raise TypeError("Invalid value type") + + # Recurse into dicts + elif type(value) == dict: + data[key] = combine_with_skeleton(data[key], value) + + return data + + +# Test cases +if __name__ == "__main__": + d1 = { + "a": 5 + } + s1 = { + "a": 7, + "b": 2 + } + r1 = { + "a": 5, + "b": 2 + } + + assert combine_with_skeleton(d1, s1) == r1 diff --git a/backups/app/specs/specs.py b/backups/app/specs/specs.py new file mode 100644 index 0000000..3f64283 --- /dev/null +++ b/backups/app/specs/specs.py @@ -0,0 +1,119 @@ +from pathlib import Path +import os + + +class Spec: + def __init__(self, name, destination, limit, title, events=None): + self.name = name + self.destination = Path(destination) + self.limit = limit + self.title = title + self.events = ["success"] if events is None else events + + def to_dict(self): + return { + "name": self.name, + "destination": str(self.destination), + "limit": self.limit, + "notify": { + "title": self.title, + "events": self.events + } + } + + def backup(self): + raise NotImplementedError() + + def notify(status_code): + if status_code: + if "failure" not in self.events: + return + + message = "backup for {} failed.".format(self.name) + + else: + if "success" not in self.events: + return + + message = "backup for {} succeeded.".format(self.name) + + # Read API key from env vars + try: + key = os.environ["IFTTT_API_KEY"] + + # Don't send notification if there's not API key defined + except KeyError: + return + + url = "https://maker.ifttt.com/trigger/{}/with/key/{}".format( + "phone_notifications", + key + ) + + data = { + "value1": self.title, + "value2": message + } + + requests.post(url, data=data) + + def get_filename(self): + return '{}_{}.tar.gz'.format( + self.name, + datetime.now().strftime('%Y-%m-%d_%H-%M-%S') + ) + + @staticmethod + def from_dict(name, data) -> "Specification": + if data.get("volume", False): + return VolumeSpec.from_dict(name, data) + + return DirSpec.from_dict(name, data) + + @staticmethod + def from_file(path: str): + with open(path, 'r') as yaml_file: + data = yaml.load(yaml_file, Loader=yaml.Loader) + + return [Spec.from_dict(name, info) + for name, info in data["specs"].items()] + + + +class DirSpec(Spec): + def __init__(self, name, source, destination, limit, title, events=None): + super().__init__(name, destination, limit, title, events) + + self.source = Path(source) + + def backup(self): + tarballs = self.destination.glob('*.tar.gz') + + # Remove redundant tarballs + if len(tarballs) >= self.limit: + for path in tarballs[self.limit - 1:]: + path.unlink() + + # Create new tarball + status_code = os.system( + "tar -C '{}' -czf '{}' -- .".format( + self.source, + self.destination / self.get_filename() + ) + ) + + self.notify(status_code) + + @staticmethod + def from_dict(name, data): + return DirSpec( + name, + data["source"], + data["destination"], + data["limit"], + data["notify"]["title"], + data["notify"]["events"] + ) + +class VolumeSpec(Spec): + pass diff --git a/backups/backups.yaml.example b/backups/backups.yaml.example new file mode 100644 index 0000000..812608e --- /dev/null +++ b/backups/backups.yaml.example @@ -0,0 +1,13 @@ +specs: + test-spec: + source: '/some/path' + destination: '/some/other/path' + limit: 7 + notify: + events: + - 'failure' + + test-2: + source: '/path/to' + destination: '/to/some/other/path' + limit: 2 diff --git a/backups/install.sh b/backups/install.sh new file mode 100755 index 0000000..c759087 --- /dev/null +++ b/backups/install.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Zip app +(cd app && zip -r ../app.zip *) + +pwd + +# Add shebang to top of file +echo "#!/usr/bin/env python3" | cat - app.zip > backup_tool +chmod a+x backup_tool + +# Move executable over +mv backup_tool /usr/local/bin + +# Remove zip +rm app.zip