Clean up

byostorage/apps.py

@@ -4,3 +4,4 @@ from django.apps import AppConfig
 class BYOStorageConfig(AppConfig):
     name = 'byostorage'
     verbose_name = 'Bring your own storage'
+    default_auto_field = 'django.db.models.AutoField'

byostorage/local_settings.py

@@ -5,11 +5,15 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 
 MEDIA_ROOT = "media"
 
+DEBUG = True
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
 INSTALLED_APPS = [
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'byostorage'
 ]
+
 SECRET_KEY = 'shh!'
 
 DATABASES = {

byostorage/migrations/0004_auto_20210917_0000.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.7 on 2021-09-17 00:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('byostorage', '0003_auto_20210323_1047'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='userstorage',
+            name='id',
+        ),
+        migrations.AlterField(
+            model_name='userstorage',
+            name='name',
+            field=models.SlugField(help_text='Storage tag', max_length=20, primary_key=True, serialize=False),
+        ),
+    ]

byostorage/models.py

@@ -1,9 +1,15 @@
 from django.conf import settings
 from django.db import models
-from django.core.files.storage import get_storage_class, FileSystemStorage
+from django.db.models.fields.files import FieldFile
+from django.core.exceptions import ObjectDoesNotExist
+from django.core.files.storage import Storage, get_storage_class
 from django.core.exceptions import ValidationError
 import json
 
+import logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
 STORAGE_CLASSES = getattr(settings, 'STORAGE_CLASSES', [])
 
 STORAGE_CLASSES.extend([
@@ -11,7 +17,7 @@ STORAGE_CLASSES.extend([
 ])
 
 try:
-    import storages
+    import storages.backends
     STORAGE_CLASSES.append('storages.backends.s3boto3.S3Boto3Storage')
 except ImportError:
     pass
@@ -24,6 +30,15 @@ def validate_json(value):
     except json.JSONDecodeError as e:
         raise ValidationError(e.msg)
 
+def resolve_instance(obj, relation):
+    for p in relation.split('__'):
+        obj = getattr(obj, p)
+
+        if callable(obj):
+            obj = obj()
+
+    return obj
+
 class UserStorage(models.Model):
     """ A user defined storage
     """
@@ -31,7 +46,7 @@ class UserStorage(models.Model):
             on_delete=models.CASCADE,
             null=True, blank=True)
     name = models.SlugField(max_length=20,
-            unique=True,
+            primary_key=True,
             help_text="Storage tag")
     storage = models.CharField(max_length=255,
             choices=STORAGE_CLASS_OPTIONS,
@@ -75,4 +90,45 @@ class UserStorage(models.Model):
         super(UserStorage, self).save(*args, **kwargs)
 
     def __str__(self):
-        return self.name
+        return self.name
+
+class BYOFieldFile(FieldFile):
+    
+    def __init__(self, instance, field, name):
+        logger.debug("BYOFieldFile(%r, %r, %r)", instance, field, name)
+        super().__init__(instance, field, name)
+        try:
+            self.storage = field.get_storage(instance)
+            logger.debug("Using BYO storage: %r", self.storage)
+        except ObjectDoesNotExist:
+            logger.debug("Unable to select BYO storage")
+            self.storage = None # trigger error if try to save etc
+
+class BYOStorageField(models.FileField):
+    attr_class = BYOFieldFile
+
+    def __init__(self, storage_instance, *args, **kwargs):
+        self.storage_instance = storage_instance
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        args = [self.storage_instance] + args
+        return name, path, args, kwargs
+
+    def get_storage(self, instance):
+
+        if callable(self.storage_instance):
+            storage = self.storage_instance(instance)
+        else:
+            storage = resolve_instance(instance, self.storage_instance)
+        
+        if not isinstance(storage, Storage):
+            raise RuntimeError("Not a storage instance")
+
+        return storage
+
+    #def formfield(self, **kwargs):
+    #    return BYOStorageFormField(**kwargs)
+
+

byostorage/tests/test_byo_storage.py

@@ -2,46 +2,17 @@ from django.test import TestCase
 from django.core.files.storage import FileSystemStorage
 from django.core.exceptions import ValidationError
 
-from .multi import MultiStorage
+from byostorage.models import UserStorage
-from .models import UserStorage
+from byostorage.user import BYOStorage
-from .user import BYOStorage
+
-from .http import HTTPStorage
+#from example.models import LinkedStorageTestModel
+
+import tempfile
+import os.path
 
 #import logging
 #logging.basicConfig(level=logging.DEBUG)
 
-class MultiStorageTestCase(TestCase):
-
-    def test_storage_selection(self):
-        ms = MultiStorage({'one': {'location': 'storage/one'}, 'two': {}})
-        self.assertEqual(str(ms), "<MultiStorage: 2 storages>")
-
-        # use the default if nothing set
-        default = ms.get_storage('foo')
-        self.assertIsInstance(default, FileSystemStorage)
-        self.assertEqual(default.base_location, 'media')
-
-        # get first instance
-        one = ms.get_storage('one')
-        self.assertEqual(one.base_location, 'storage/one')
-
-    def test_storage_proxies(self):
-        ms = MultiStorage({
-            'one': {'location': 'storage/one', 'base_url': 'http://one'},
-            'two': {'location': 'storage/two', 'base_url': 'http://two'}
-        })
-
-
-        self.assertEqual(ms.url("one:foo.txt"), 'http://one/foo.txt')
-        self.assertEqual(ms.url("two:foo.txt"), 'http://two/foo.txt')
-
-    def test_with_http(self):
-        ms = MultiStorage({
-            'https': {'storage': 'byostorage.http.HTTPStorage', 'protocol': 'https'}
-        })
-
-        self.assertEqual(ms.url('https://google.com'), 'https://google.com')
-
 class UserStorageTestCase(TestCase):
 
     def test_construction(self):
@@ -63,10 +34,10 @@ class UserStorageTestCase(TestCase):
         instance = UserStorage.objects.create(name='one', storage='storages.backends.s3boto3.S3Boto3Storage',
                 settings_data='''
 {
-  "access_key": "polyphonic_test_key",
+  "access_key": "minioadmin",
-  "secret_key": "polyphonic_secret",
+  "secret_key": "minioadmin",
   "endpoint_url": "http://localhost:9000",
-  "bucket_name": "personal"
+  "bucket_name": "byostorage_test"
 }
 ''')
 
@@ -76,10 +47,10 @@ class UserStorageTestCase(TestCase):
             UserStorage.objects.create(name='one', storage='storages.backends.s3boto3.S3Boto3Storage',
                    settings_data='''
 {
-  "access_key": "polyphonic_test_key",
+  "access_key": "minioadmin",
   "secret_key": "the_wrong_secret",
   "endpoint_url": "http://localhost:9000",
-  "bucket_name": "missing"
+  "bucket_name": "byostorage_test"
 }
 ''')
             self.fail("Should have raised an exception")
@@ -113,28 +84,29 @@ class BYOStorageTestCase(TestCase):
         one = storage.get_storage('one')
         self.assertEqual(one.base_location, 'other/one')
 
+"""
+class BYOStorageFieldTestCase(TestCase):
 
-class HTTPStorageTestCase(TestCase):
+    def setUp(self):
+        self.test_dir = tempfile.TemporaryDirectory()
+        self.one = UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage',
+            settings_data = '{"location": "%s/one"}' % self.test_dir.name)
 
-    def test_url(self):
+    def tearDown(self):
-        s = HTTPStorage()
+        self.test_dir.cleanup()
 
-        self.assertEqual(s.url('//google.com'), 'https://google.com')
+    def test_linked_storage(self):
+        
+        from django.core.files.uploadedfile import UploadedFile
 
-    def test_exists(self):
+        with open(__file__, 'rb') as f:
-        s = HTTPStorage()
+            result = LinkedStorageTestModel.objects.create(name="first", my_storage_id=self.one.pk, my_file=UploadedFile(f))
 
-        self.assertTrue(s.exists('//gitea.tfconsulting.com.au/tris'))
+        expected = os.path.basename(__file__)
-        self.assertFalse(s.exists('//gitea.tfconsulting.com.au/foo.txt'))
 
-    def test_save(self):
+        self.assertEqual(result.my_file.name, expected)
-        s = HTTPStorage()
+        self.assertGreater(result.my_file.size, 0)
-        with self.assertRaisesMessage(NotImplementedError, "Unable to save to web locations"):
-            s.save('//gitea.tfconsulting.com.au/foo', 'Some content')
 
-    def test_open(self):
+        self.assertEqual(os.listdir(os.path.join(self.test_dir.name, 'one')), [expected])
-        s = HTTPStorage()
-        with s.open('//gitea.tfconsulting.com.au', 'rb') as f:
-            data = f.read()
 
-        self.assertTrue(len(data) > 4000)
+"""

byostorage/tests/test_http_storage.py

@@ -0,0 +1,28 @@
+from django.test import TestCase
+
+from byostorage.http import HTTPStorage
+
+class HTTPStorageTestCase(TestCase):
+
+    def test_url(self):
+        s = HTTPStorage()
+
+        self.assertEqual(s.url('//google.com'), 'https://google.com')
+
+    def test_exists(self):
+        s = HTTPStorage()
+
+        self.assertTrue(s.exists('//gitea.tfconsulting.com.au/tris'))
+        self.assertFalse(s.exists('//gitea.tfconsulting.com.au/foo.txt'))
+
+    def test_save(self):
+        s = HTTPStorage()
+        with self.assertRaisesMessage(NotImplementedError, "Unable to save to web locations"):
+            s.save('//gitea.tfconsulting.com.au/foo', 'Some content')
+
+    def test_open(self):
+        s = HTTPStorage()
+        with s.open('//gitea.tfconsulting.com.au', 'rb') as f:
+            data = f.read()
+
+        self.assertTrue(len(data) > 4000)

byostorage/tests/test_multi_storage.py

@@ -0,0 +1,36 @@
+from django.test import TestCase
+from byostorage.multi import MultiStorage
+from django.core.files.storage import FileSystemStorage
+
+
+class MultiStorageTestCase(TestCase):
+
+    def test_storage_selection(self):
+        ms = MultiStorage({'one': {'location': 'storage/one'}, 'two': {}})
+        self.assertEqual(str(ms), "<MultiStorage: 2 storages>")
+
+        # use the default if nothing set
+        default = ms.get_storage('foo')
+        self.assertIsInstance(default, FileSystemStorage)
+        self.assertEqual(default.base_location, 'media')
+
+        # get first instance
+        one = ms.get_storage('one')
+        self.assertEqual(one.base_location, 'storage/one')
+
+    def test_storage_proxies(self):
+        ms = MultiStorage({
+            'one': {'location': 'storage/one', 'base_url': 'http://one'},
+            'two': {'location': 'storage/two', 'base_url': 'http://two'}
+        })
+
+
+        self.assertEqual(ms.url("one:foo.txt"), 'http://one/foo.txt')
+        self.assertEqual(ms.url("two:foo.txt"), 'http://two/foo.txt')
+
+    def test_with_http(self):
+        ms = MultiStorage({
+            'https': {'storage': 'byostorage.http.HTTPStorage', 'protocol': 'https'}
+        })
+
+        self.assertEqual(ms.url('https://google.com'), 'https://google.com')

byostorage/user.py

@@ -6,7 +6,6 @@ from .multi import MultiStorage
 class BYOStorage(MultiStorage):
     ''' Database driven Bring-Your-Own-Storage
 
-    Multiple storages can 
     '''
 
     def __init__(self, config=None):