diff --git a/byostorage/http.py b/byostorage/http.py new file mode 100644 index 0000000..cc69134 --- /dev/null +++ b/byostorage/http.py @@ -0,0 +1,26 @@ +from django.core.files.storage import Storage +import requests + +class HTTPStorage(Storage): + + def __init__(self, protocol='https'): + self.protocol = protocol + + def _open(self, name, mode='rb'): + if mode != 'rb': + raise IOError("Mode must be 'rb'") + + r = requests.get(self.url(name), stream=True) + r.raise_for_status() + + return r.raw + + def _save(self, name, content): + raise NotImplementedError("Unable to save to web locations") + + def url(self, name): + return f"{self.protocol}:{name}" + + def exists(self, name): + r = requests.head(self.url(name)) + return r.status_code == 200 \ No newline at end of file diff --git a/byostorage/local_settings.py b/byostorage/local_settings.py index e963dfd..46ff61a 100644 --- a/byostorage/local_settings.py +++ b/byostorage/local_settings.py @@ -1,6 +1,20 @@ +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + +MEDIA_ROOT = "media" + INSTALLED_APPS = [ 'django.contrib.auth', 'django.contrib.contenttypes', 'byostorage' ] -SECRET_KEY = 'shh!' \ No newline at end of file +SECRET_KEY = 'shh!' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} \ No newline at end of file diff --git a/byostorage/models.py b/byostorage/models.py index 9de8530..de9ccf6 100644 --- a/byostorage/models.py +++ b/byostorage/models.py @@ -1,6 +1,7 @@ from django.conf import settings from django.db import models from django.core.files.storage import get_storage_class +from django.core.exceptions import ValidationError import json STORAGE_CLASSES = getattr(settings, 'STORAGE_CLASSES', []) @@ -17,6 +18,12 @@ except ImportError: STORAGE_CLASS_OPTIONS = [ (x, x) for x in STORAGE_CLASSES ] +def validate_json(value): + try: + json.loads(value) + except json.JSONDecodeError as e: + raise ValidationError(e.msg) + class UserStorage(models.Model): """ A user defined storage """ @@ -29,7 +36,9 @@ class UserStorage(models.Model): storage = models.CharField(max_length=255, choices=STORAGE_CLASS_OPTIONS, help_text="Storage class for this instance") - settings_data = models.TextField(help_text="JSON dict with key/value settings") + settings_data = models.TextField( + validators=[validate_json], + help_text="JSON dict with key/value settings") def instance(self): return get_storage_class(self.storage)(**self.settings) @@ -38,11 +47,20 @@ class UserStorage(models.Model): def settings(self): if not self.settings_data: return {} - return json.loads(self.settings_data) + try: + return json.loads(self.settings_data) + except Exception as e: + raise ValueError("Error in settings for storage '{0}' [{1}]".format(self.name, e)) + + def test_connection(self): + # just do something that requires connection + self.instance().exists('ping.txt') def save(self, *args, **kwargs): # check the settings are valid - self.settings + validate_json(self.settings_data) + self.test_connection() + super(UserStorage, self).save(*args, **kwargs) def __str__(self): diff --git a/byostorage/multi.py b/byostorage/multi.py index 0923626..278b953 100644 --- a/byostorage/multi.py +++ b/byostorage/multi.py @@ -1,4 +1,4 @@ -from django.core.files.storage import Storage +from django.core.files.storage import Storage, default_storage, get_storage_class from django.conf import settings from django.utils.deconstruct import deconstructible @@ -24,8 +24,12 @@ class MultiStorage(Storage): def get_storage(self, name): if name not in self._cache: - config = self.config[name] - self._cache[name] = get_storage_class(config['storage'])(**config['settings']) + try: + config = self.config[name] + except KeyError: + return default_storage + klass = get_storage_class(config.pop('storage', None)) + self._cache[name] = klass(**config) return self._cache[name] def split(self, name): @@ -77,21 +81,5 @@ class MultiStorage(Storage): def get_modified_time(self, name): return self._proxy('get_modified_time', name) - -class BYOStorage(MultiStorage): - ''' Database driven Bring-Your-Own-Storage - - Multiple storages can - ''' - - def get_storage(self, name): - if name not in self._cache: - - # use storage from config by default - if name in self.config: - return super(BYOStorage, self).get_storage(name) - - from .models import UserStorage - obj = UserStorage.objects.get(name=name) - self._cache[name] = obj.instance() - return self._cache[name] \ No newline at end of file + def __str__(self): + return "".format(len(self.config)) diff --git a/byostorage/tests.py b/byostorage/tests.py new file mode 100644 index 0000000..b8d5152 --- /dev/null +++ b/byostorage/tests.py @@ -0,0 +1,120 @@ +from django.test import TestCase +from django.core.files.storage import FileSystemStorage +from django.core.exceptions import ValidationError + +from .multi import MultiStorage +from .models import UserStorage +from .user import BYOStorage +from .http import HTTPStorage + +class MultiStorageTestCase(TestCase): + + def test_storage_selection(self): + ms = MultiStorage({'one': {'location': 'storage/one'}, 'two': {}}) + self.assertEqual(str(ms), "") + + # use the default if nothing set + default = ms.get_storage('foo') + self.assertIsInstance(default, FileSystemStorage) + self.assertEqual(default.base_location, 'media') + + # get first instance + one = ms.get_storage('one') + self.assertEqual(one.base_location, 'storage/one') + + def test_storage_proxies(self): + ms = MultiStorage({ + 'one': {'location': 'storage/one', 'base_url': 'http://one'}, + 'two': {'location': 'storage/two', 'base_url': 'http://two'} + }) + + + self.assertEqual(ms.url("one:foo.txt"), 'http://one/foo.txt') + self.assertEqual(ms.url("two:foo.txt"), 'http://two/foo.txt') + + def test_with_http(self): + ms = MultiStorage({ + 'https': {'storage': 'byostorage.http.HTTPStorage', 'protocol': 'https'} + }) + + self.assertEqual(ms.url('https://google.com'), 'https://google.com') + +class UserStorageTestCase(TestCase): + + def test_construction(self): + us = UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage', + settings_data='{"location": "storage/one"}') + + self.assertEqual(str(us), 'one') + + storage = us.instance() + self.assertIsInstance(storage, FileSystemStorage) + self.assertEqual(storage.base_location, 'storage/one') + + def test_bad_json(self): + with self.assertRaisesMessage(ValidationError, "Unterminated string"): + UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage', + settings_data='{"location": "foo}') + + def test_bad_settings(self): + UserStorage.objects.create(name='one', storage='storages.backends.s3boto3.S3Boto3Storage', + settings_data=''' +{ + "access_key": "polyphonic_test_key", + "secret_key": "polyphonic_secret", + "endpoint_url": "http://localhost:9000", + "bucket_name": "personal" +} +''') + +class BYOStorageTestCase(TestCase): + + def setUp(self): + UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage', + settings_data='{"location": "storage/one"}') + + def test_pickup_changes(self): + storage = BYOStorage() + one = storage.get_storage('one') + self.assertEqual(one.base_location, 'storage/one') + + # if not already in cache then no issues + UserStorage.objects.create(name='two', storage='django.core.files.storage.FileSystemStorage', + settings_data='{"location": "storage/two"}') + two = storage.get_storage('two') + self.assertEqual(two.base_location, 'storage/two') + + # modify the settings + o = UserStorage.objects.get(name='one') + o.settings_data = '{"location": "other/one"}' + o.save() + + # signal should have fired and invalidated 'one' + one = storage.get_storage('one') + self.assertEqual(one.base_location, 'other/one') + + +class HTTPStorageTestCase(TestCase): + + def test_url(self): + s = HTTPStorage() + + self.assertEqual(s.url('//google.com'), 'https://google.com') + + def test_exists(self): + s = HTTPStorage() + + self.assertTrue(s.exists('//gitea.tfconsulting.com.au/tris')) + self.assertFalse(s.exists('//gitea.tfconsulting.com.au/foo.txt')) + + def test_save(self): + s = HTTPStorage() + with self.assertRaisesMessage(NotImplementedError, "Unable to save to web locations"): + s.save('//gitea.tfconsulting.com.au/foo', 'Some content') + + def test_open(self): + s = HTTPStorage() + with s.open('//gitea.tfconsulting.com.au', 'rb') as f: + data = f.read() + + self.assertTrue(len(data) > 4000) \ No newline at end of file diff --git a/byostorage/user.py b/byostorage/user.py new file mode 100644 index 0000000..8b76c1d --- /dev/null +++ b/byostorage/user.py @@ -0,0 +1,33 @@ +from django.db.models.signals import post_save + +from .models import UserStorage +from .multi import MultiStorage + +class BYOStorage(MultiStorage): + ''' Database driven Bring-Your-Own-Storage + + Multiple storages can + ''' + + def __init__(self, config=None): + super(BYOStorage, self).__init__(config) + + post_save.connect(self.handle_change, sender=UserStorage) + + + def handle_change(self, instance, **kwargs): + try: + del(self._cache[instance.name]) + except KeyError: + pass + + def get_storage(self, name): + if name not in self._cache: + + # use storage from config by default + if name in self.config: + return super(BYOStorage, self).get_storage(name) + + obj = UserStorage.objects.get(name=name) + self._cache[name] = obj.instance() + return self._cache[name] \ No newline at end of file