Wrote tests and fixed a few issues

byostorage/http.py

@@ -0,0 +1,26 @@
+from django.core.files.storage import Storage
+import requests
+
+class HTTPStorage(Storage):
+
+    def __init__(self, protocol='https'):
+        self.protocol = protocol
+
+    def _open(self, name, mode='rb'):
+        if mode != 'rb':
+            raise IOError("Mode must be 'rb'")
+
+        r = requests.get(self.url(name), stream=True)
+        r.raise_for_status()
+
+        return r.raw
+
+    def _save(self, name, content):
+        raise NotImplementedError("Unable to save to web locations")
+
+    def url(self, name):
+        return f"{self.protocol}:{name}"
+
+    def exists(self, name):
+        r = requests.head(self.url(name))
+        return r.status_code == 200

byostorage/local_settings.py

@@ -1,6 +1,20 @@
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+MEDIA_ROOT = "media"
+
 INSTALLED_APPS = [
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'byostorage'
 ]
-SECRET_KEY = 'shh!'
+SECRET_KEY = 'shh!'
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': BASE_DIR / 'db.sqlite3',
+    }
+}

byostorage/models.py

@@ -1,6 +1,7 @@
 from django.conf import settings
 from django.db import models
 from django.core.files.storage import get_storage_class
+from django.core.exceptions import ValidationError
 import json
 
 STORAGE_CLASSES = getattr(settings, 'STORAGE_CLASSES', [])
@@ -17,6 +18,12 @@ except ImportError:
 
 STORAGE_CLASS_OPTIONS = [ (x, x) for x in STORAGE_CLASSES ]
 
+def validate_json(value):
+    try:
+        json.loads(value)
+    except json.JSONDecodeError as e:
+        raise ValidationError(e.msg)
+
 class UserStorage(models.Model):
     """ A user defined storage
     """
@@ -29,7 +36,9 @@ class UserStorage(models.Model):
     storage = models.CharField(max_length=255,
             choices=STORAGE_CLASS_OPTIONS,
             help_text="Storage class for this instance")
-    settings_data = models.TextField(help_text="JSON dict with key/value settings")
+    settings_data = models.TextField(
+            validators=[validate_json],
+            help_text="JSON dict with key/value settings")
 
     def instance(self):
         return get_storage_class(self.storage)(**self.settings)
@@ -38,11 +47,20 @@ class UserStorage(models.Model):
     def settings(self):
         if not self.settings_data:
             return {}
-        return json.loads(self.settings_data)
+        try:
+            return json.loads(self.settings_data)
+        except Exception as e:
+            raise ValueError("Error in settings for storage '{0}' [{1}]".format(self.name, e))
+
+    def test_connection(self):
+        # just do something that requires connection
+        self.instance().exists('ping.txt')
 
     def save(self, *args, **kwargs):
         # check the settings are valid
-        self.settings
+        validate_json(self.settings_data)
+        self.test_connection()
+
         super(UserStorage, self).save(*args, **kwargs)
 
     def __str__(self):

byostorage/multi.py

@@ -1,4 +1,4 @@
-from django.core.files.storage import Storage
+from django.core.files.storage import Storage, default_storage, get_storage_class
 from django.conf import settings
 from django.utils.deconstruct import deconstructible
 
@@ -24,8 +24,12 @@ class MultiStorage(Storage):
 
     def get_storage(self, name):
         if name not in self._cache:
-            config = self.config[name]
-            self._cache[name] = get_storage_class(config['storage'])(**config['settings'])
+            try:
+                config = self.config[name]
+            except KeyError:
+                return default_storage
+            klass = get_storage_class(config.pop('storage', None))
+            self._cache[name] = klass(**config)
         return self._cache[name]
 
     def split(self, name):
@@ -77,21 +81,5 @@ class MultiStorage(Storage):
     def get_modified_time(self, name):
         return self._proxy('get_modified_time', name)
 
-
-class BYOStorage(MultiStorage):
-    ''' Database driven Bring-Your-Own-Storage
-
-    Multiple storages can 
-    '''
-
-    def get_storage(self, name):
-        if name not in self._cache:
-
-            # use storage from config by default
-            if name in self.config:
-                return super(BYOStorage, self).get_storage(name)
-
-            from .models import UserStorage
-            obj = UserStorage.objects.get(name=name)
-            self._cache[name] = obj.instance()
-        return self._cache[name]
+    def __str__(self):
+        return "<MultiStorage: {0} storages>".format(len(self.config))

byostorage/tests.py

@@ -0,0 +1,120 @@
+from django.test import TestCase
+from django.core.files.storage import FileSystemStorage
+from django.core.exceptions import ValidationError
+
+from .multi import MultiStorage
+from .models import UserStorage
+from .user import BYOStorage
+from .http import HTTPStorage
+
+class MultiStorageTestCase(TestCase):
+
+    def test_storage_selection(self):
+        ms = MultiStorage({'one': {'location': 'storage/one'}, 'two': {}})
+        self.assertEqual(str(ms), "<MultiStorage: 2 storages>")
+
+        # use the default if nothing set
+        default = ms.get_storage('foo')
+        self.assertIsInstance(default, FileSystemStorage)
+        self.assertEqual(default.base_location, 'media')
+
+        # get first instance
+        one = ms.get_storage('one')
+        self.assertEqual(one.base_location, 'storage/one')
+
+    def test_storage_proxies(self):
+        ms = MultiStorage({
+            'one': {'location': 'storage/one', 'base_url': 'http://one'},
+            'two': {'location': 'storage/two', 'base_url': 'http://two'}
+        })
+
+
+        self.assertEqual(ms.url("one:foo.txt"), 'http://one/foo.txt')
+        self.assertEqual(ms.url("two:foo.txt"), 'http://two/foo.txt')
+
+    def test_with_http(self):
+        ms = MultiStorage({
+            'https': {'storage': 'byostorage.http.HTTPStorage', 'protocol': 'https'}
+        })
+
+        self.assertEqual(ms.url('https://google.com'), 'https://google.com')
+
+class UserStorageTestCase(TestCase):
+
+    def test_construction(self):
+        us = UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage',
+                settings_data='{"location": "storage/one"}')
+
+        self.assertEqual(str(us), 'one')
+
+        storage = us.instance()
+        self.assertIsInstance(storage, FileSystemStorage)
+        self.assertEqual(storage.base_location, 'storage/one')
+
+    def test_bad_json(self):
+        with self.assertRaisesMessage(ValidationError, "Unterminated string"):
+            UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage',
+            settings_data='{"location": "foo}')
+
+    def test_bad_settings(self):
+        UserStorage.objects.create(name='one', storage='storages.backends.s3boto3.S3Boto3Storage',
+                settings_data='''
+{
+  "access_key": "polyphonic_test_key",
+  "secret_key": "polyphonic_secret",
+  "endpoint_url": "http://localhost:9000",
+  "bucket_name": "personal"
+}
+''')
+
+class BYOStorageTestCase(TestCase):
+
+    def setUp(self):
+        UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage',
+                settings_data='{"location": "storage/one"}')
+    
+    def test_pickup_changes(self):
+        storage = BYOStorage()
+        one = storage.get_storage('one')
+        self.assertEqual(one.base_location, 'storage/one')
+
+        # if not already in cache then no issues
+        UserStorage.objects.create(name='two', storage='django.core.files.storage.FileSystemStorage',
+                settings_data='{"location": "storage/two"}')
+        two = storage.get_storage('two')
+        self.assertEqual(two.base_location, 'storage/two')
+
+        # modify the settings
+        o = UserStorage.objects.get(name='one')
+        o.settings_data = '{"location": "other/one"}'
+        o.save()
+
+        # signal should have fired and invalidated 'one'
+        one = storage.get_storage('one')
+        self.assertEqual(one.base_location, 'other/one')
+
+
+class HTTPStorageTestCase(TestCase):
+
+    def test_url(self):
+        s = HTTPStorage()
+
+        self.assertEqual(s.url('//google.com'), 'https://google.com')
+
+    def test_exists(self):
+        s = HTTPStorage()
+
+        self.assertTrue(s.exists('//gitea.tfconsulting.com.au/tris'))
+        self.assertFalse(s.exists('//gitea.tfconsulting.com.au/foo.txt'))
+
+    def test_save(self):
+        s = HTTPStorage()
+        with self.assertRaisesMessage(NotImplementedError, "Unable to save to web locations"):
+            s.save('//gitea.tfconsulting.com.au/foo', 'Some content')
+
+    def test_open(self):
+        s = HTTPStorage()
+        with s.open('//gitea.tfconsulting.com.au', 'rb') as f:
+            data = f.read()
+
+        self.assertTrue(len(data) > 4000)

byostorage/user.py

@@ -0,0 +1,33 @@
+from django.db.models.signals import post_save
+
+from .models import UserStorage
+from .multi import MultiStorage
+
+class BYOStorage(MultiStorage):
+    ''' Database driven Bring-Your-Own-Storage
+
+    Multiple storages can 
+    '''
+
+    def __init__(self, config=None):
+        super(BYOStorage, self).__init__(config)
+
+        post_save.connect(self.handle_change, sender=UserStorage)
+
+
+    def handle_change(self, instance, **kwargs):
+        try:
+            del(self._cache[instance.name])
+        except KeyError:
+            pass
+
+    def get_storage(self, name):
+        if name not in self._cache:
+
+            # use storage from config by default
+            if name in self.config:
+                return super(BYOStorage, self).get_storage(name)
+
+            obj = UserStorage.objects.get(name=name)
+            self._cache[name] = obj.instance()
+        return self._cache[name]