Updated cache to not use get_storage_class

README.rst

@@ -23,3 +23,8 @@ Usage
      photo = models.FileField(
          storage=BYOStorage(),
          upload_to=generate_filename)
+
+Notes
+=====
+
+Boto3 client sometimes doesn't respect the `region_name` so it may be neccesary to set `endpoint_url: https://https://s3-<region>.amazonaws.com`

byostorage/__init__.py

@@ -1,2 +1,2 @@
-__version__ = 0.1
-default_app_config = 'byostorage.apps.BYOStorageConfig'
+__version__ = '0.2.4'
+default_app_config = 'byostorage.apps.BYOStorageConfig'

byostorage/apps.py

@@ -4,3 +4,4 @@ from django.apps import AppConfig
 class BYOStorageConfig(AppConfig):
     name = 'byostorage'
     verbose_name = 'Bring your own storage'
+    default_auto_field = 'django.db.models.AutoField'

byostorage/cached.py

@@ -1,5 +1,6 @@
-from django.core.files.storage import Storage, get_storage_class
+from django.core.files.storage import Storage, DefaultStorage
 from django.utils.deconstruct import deconstructible
+from django.utils.module_loading import import_string
 from django.conf import settings
 from hashlib import sha1
 import os.path
@@ -15,11 +16,14 @@ class CachedStorage(Storage):
     
     def __init__(self, remote=None, cachedir=None, expires=600):
         if not remote:
-            remote = getattr(settings, 'CACHED_STORAGE_REMOTE', None)
+            remote = getattr(settings, 'CACHED_STORAGE_REMOTE', DefaultStorage)
         if not cachedir:
             cachedir = getattr(settings, 'CACHED_STORAGE_DIR', 'cache')
-        
-        self.remote = get_storage_class(remote)()
+
+        if isinstance(remote, Storage):
+            self.remote = remote
+        else:
+            self.remote = import_string(remote)()
         self.cachedir = cachedir
         self.expires = expires
 
@@ -52,16 +56,20 @@ class CachedStorage(Storage):
 
     def _open(self, name, mode='rb'):
         assert 'r' in mode, "Can only open for reading"
-        p = self._cached(name)
+        p = self.path(name)
         return open(p, mode)
 
     def path(self, name):
-        return self._cached(name)
+        try:
+            return self.remote.path(name)
+        except NotImplementedError:
+            return self._cached(name)
 
     def save(self, name, content, max_length=None):
         p = self._filepath(name)
         if os.path.exists(p):
             os.unlink(p)
+        # TODO: cache content to p
         return self.remote.save(name, content, max_length)
 
     def delete(self, name):

byostorage/local_settings.py

@@ -5,11 +5,15 @@ BASE_DIR = Path(__file__).resolve().parent.parent
 
 MEDIA_ROOT = "media"
 
+DEBUG = True
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
 INSTALLED_APPS = [
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'byostorage'
 ]
+
 SECRET_KEY = 'shh!'
 
 DATABASES = {

byostorage/migrations/0004_auto_20210917_0000.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.7 on 2021-09-17 00:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('byostorage', '0003_auto_20210323_1047'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='userstorage',
+            name='id',
+        ),
+        migrations.AlterField(
+            model_name='userstorage',
+            name='name',
+            field=models.SlugField(help_text='Storage tag', max_length=20, primary_key=True, serialize=False),
+        ),
+    ]

byostorage/migrations/0005_alter_userstorage_storage.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.7 on 2022-09-09 23:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('byostorage', '0004_auto_20210917_0000'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='userstorage',
+            name='storage',
+            field=models.CharField(choices=[('django.core.files.storage.FileSystemStorage', 'django.core.files.storage.FileSystemStorage')], help_text='Storage class for this instance', max_length=255),
+        ),
+    ]

byostorage/migrations/0006_alter_userstorage_settings_data.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.7 on 2022-09-10 00:10
+
+import byostorage.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('byostorage', '0005_alter_userstorage_storage'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='userstorage',
+            name='settings_data',
+            field=models.TextField(default='{}', help_text='JSON dict with key/value settings', validators=[byostorage.models.validate_json]),
+        ),
+    ]

byostorage/models.py

@@ -1,9 +1,16 @@
 from django.conf import settings
 from django.db import models
-from django.core.files.storage import get_storage_class, FileSystemStorage
+from django.db.models.fields.files import FieldFile
+from django.core.exceptions import ObjectDoesNotExist
+from django.core.files.storage import Storage
 from django.core.exceptions import ValidationError
+from django.utils.module_loading import import_string
 import json
 
+import logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
 STORAGE_CLASSES = getattr(settings, 'STORAGE_CLASSES', [])
 
 STORAGE_CLASSES.extend([
@@ -11,8 +18,16 @@ STORAGE_CLASSES.extend([
 ])
 
 try:
-    import storages
-    STORAGE_CLASSES.append('storages.backends.s3boto3.S3Boto3Storage')
+    import storages.backends
+
+    STORAGE_CLASSES.extend([
+        'storages.backends.s3boto3.S3Boto3Storage',
+        'storages.backends.azure_storage.AzureStorage',
+        'storages.backends.dropbox.DropBoxStorage',
+        'storages.backends.ftp.FTPStorage',
+        'storages.backends.gcloud.GoogleCloudStorage',
+        'storages.backends.sftpstorage.SFTPStorage',
+    ])
 except ImportError:
     pass
 
@@ -24,6 +39,15 @@ def validate_json(value):
     except json.JSONDecodeError as e:
         raise ValidationError(e.msg)
 
+def resolve_instance(obj, relation):
+    for p in relation.split('__'):
+        obj = getattr(obj, p)
+
+        if callable(obj):
+            obj = obj()
+
+    return obj
+
 class UserStorage(models.Model):
     """ A user defined storage
     """
@@ -31,17 +55,18 @@ class UserStorage(models.Model):
             on_delete=models.CASCADE,
             null=True, blank=True)
     name = models.SlugField(max_length=20,
-            unique=True,
+            primary_key=True,
             help_text="Storage tag")
     storage = models.CharField(max_length=255,
             choices=STORAGE_CLASS_OPTIONS,
             help_text="Storage class for this instance")
     settings_data = models.TextField(
             validators=[validate_json],
+            default='{}',
             help_text="JSON dict with key/value settings")
 
     def instance(self):
-        return get_storage_class(self.storage)(**self.settings)
+        return import_string(self.storage)(**self.settings)
 
     @property
     def settings(self):
@@ -75,4 +100,45 @@ class UserStorage(models.Model):
         super(UserStorage, self).save(*args, **kwargs)
 
     def __str__(self):
-        return self.name
+        return self.name
+
+class BYOFieldFile(FieldFile):
+    
+    def __init__(self, instance, field, name):
+        logger.debug("BYOFieldFile(%r, %r, %r)", instance, field, name)
+        super().__init__(instance, field, name)
+        try:
+            self.storage = field.get_storage(instance)
+            logger.debug("Using BYO storage: %r", self.storage)
+        except ObjectDoesNotExist:
+            logger.debug("Unable to select BYO storage")
+            self.storage = None # trigger error if try to save etc
+
+class BYOStorageField(models.FileField):
+    attr_class = BYOFieldFile
+
+    def __init__(self, storage_instance, *args, **kwargs):
+        self.storage_instance = storage_instance
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        args = [self.storage_instance] + args
+        return name, path, args, kwargs
+
+    def get_storage(self, instance):
+
+        if callable(self.storage_instance):
+            storage = self.storage_instance(instance)
+        else:
+            storage = resolve_instance(instance, self.storage_instance)
+        
+        if not isinstance(storage, Storage):
+            raise RuntimeError("Not a storage instance")
+
+        return storage
+
+    #def formfield(self, **kwargs):
+    #    return BYOStorageFormField(**kwargs)
+
+

byostorage/multi.py

@@ -1,6 +1,7 @@
-from django.core.files.storage import Storage, default_storage, get_storage_class
+from django.core.files.storage import Storage, default_storage, storages
 from django.conf import settings
 from django.utils.deconstruct import deconstructible
+from django.utils.module_loading import import_string
 
 '''
 TODO: Create a signal to remove instances from cache if modified
@@ -28,7 +29,7 @@ class MultiStorage(Storage):
                 config = self.config[name]
             except KeyError:
                 return default_storage
-            klass = get_storage_class(config.pop('storage', None))
+            klass = import_string(config.pop('storage', 'django.core.files.storage.FileSystemStorage'))
             self._cache[name] = klass(**config)
         return self._cache[name]
 
@@ -45,18 +46,21 @@ class MultiStorage(Storage):
     def _proxy(self, method, name, *args, **kwargs):
         storage, p = self.split(name)
         result =  getattr(self.get_storage(storage), method)(p, *args, **kwargs)
-        logger.debug("Proxy: %s(%r, *%r, **%r) => %r", method, name, args, kwargs, result)
+        logger.debug("Proxy: %s(%r, *%r, **%r) => %r", method, p, args, kwargs, result)
         return result
 
     def _proxy_name(self, method, name, *args, **kwargs):
         storage, p = self.split(name)
         result = getattr(self.get_storage(storage), method)(p, *args, **kwargs)
-        logger.debug("Proxy: %s(%r, *%r, **%r) => %s:%s", method, name, args, kwargs, storage, result)
+        logger.debug("Proxy: %s(%r, *%r, **%r) => %s:%s", method, p, args, kwargs, storage, result)
         return self.join(storage, result)
 
     def _open(self, name, mode='rb'):
         return self._proxy('open', name, mode)
 
+    def path(self, name):
+        return self._proxy('path', name)
+
     def save(self, name, content, max_length=None):
         return self._proxy_name('save', name, content, max_length)
 

byostorage/tests/test_byo_storage.py

@@ -2,46 +2,17 @@ from django.test import TestCase
 from django.core.files.storage import FileSystemStorage
 from django.core.exceptions import ValidationError
 
-from .multi import MultiStorage
-from .models import UserStorage
-from .user import BYOStorage
-from .http import HTTPStorage
+from byostorage.models import UserStorage
+from byostorage.user import BYOStorage
+
+#from example.models import LinkedStorageTestModel
+
+import tempfile
+import os.path
 
 #import logging
 #logging.basicConfig(level=logging.DEBUG)
 
-class MultiStorageTestCase(TestCase):
-
-    def test_storage_selection(self):
-        ms = MultiStorage({'one': {'location': 'storage/one'}, 'two': {}})
-        self.assertEqual(str(ms), "<MultiStorage: 2 storages>")
-
-        # use the default if nothing set
-        default = ms.get_storage('foo')
-        self.assertIsInstance(default, FileSystemStorage)
-        self.assertEqual(default.base_location, 'media')
-
-        # get first instance
-        one = ms.get_storage('one')
-        self.assertEqual(one.base_location, 'storage/one')
-
-    def test_storage_proxies(self):
-        ms = MultiStorage({
-            'one': {'location': 'storage/one', 'base_url': 'http://one'},
-            'two': {'location': 'storage/two', 'base_url': 'http://two'}
-        })
-
-
-        self.assertEqual(ms.url("one:foo.txt"), 'http://one/foo.txt')
-        self.assertEqual(ms.url("two:foo.txt"), 'http://two/foo.txt')
-
-    def test_with_http(self):
-        ms = MultiStorage({
-            'https': {'storage': 'byostorage.http.HTTPStorage', 'protocol': 'https'}
-        })
-
-        self.assertEqual(ms.url('https://google.com'), 'https://google.com')
-
 class UserStorageTestCase(TestCase):
 
     def test_construction(self):
@@ -63,10 +34,10 @@ class UserStorageTestCase(TestCase):
         instance = UserStorage.objects.create(name='one', storage='storages.backends.s3boto3.S3Boto3Storage',
                 settings_data='''
 {
-  "access_key": "polyphonic_test_key",
-  "secret_key": "polyphonic_secret",
+  "access_key": "minioadmin",
+  "secret_key": "minioadmin",
   "endpoint_url": "http://localhost:9000",
-  "bucket_name": "personal"
+  "bucket_name": "byostorage_test"
 }
 ''')
 
@@ -76,10 +47,10 @@ class UserStorageTestCase(TestCase):
             UserStorage.objects.create(name='one', storage='storages.backends.s3boto3.S3Boto3Storage',
                    settings_data='''
 {
-  "access_key": "polyphonic_test_key",
+  "access_key": "minioadmin",
   "secret_key": "the_wrong_secret",
   "endpoint_url": "http://localhost:9000",
-  "bucket_name": "missing"
+  "bucket_name": "byostorage_test"
 }
 ''')
             self.fail("Should have raised an exception")
@@ -113,28 +84,29 @@ class BYOStorageTestCase(TestCase):
         one = storage.get_storage('one')
         self.assertEqual(one.base_location, 'other/one')
 
+"""
+class BYOStorageFieldTestCase(TestCase):
 
-class HTTPStorageTestCase(TestCase):
+    def setUp(self):
+        self.test_dir = tempfile.TemporaryDirectory()
+        self.one = UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage',
+            settings_data = '{"location": "%s/one"}' % self.test_dir.name)
 
-    def test_url(self):
-        s = HTTPStorage()
+    def tearDown(self):
+        self.test_dir.cleanup()
 
-        self.assertEqual(s.url('//google.com'), 'https://google.com')
+    def test_linked_storage(self):
+        
+        from django.core.files.uploadedfile import UploadedFile
 
-    def test_exists(self):
-        s = HTTPStorage()
+        with open(__file__, 'rb') as f:
+            result = LinkedStorageTestModel.objects.create(name="first", my_storage_id=self.one.pk, my_file=UploadedFile(f))
 
-        self.assertTrue(s.exists('//gitea.tfconsulting.com.au/tris'))
-        self.assertFalse(s.exists('//gitea.tfconsulting.com.au/foo.txt'))
+        expected = os.path.basename(__file__)
 
-    def test_save(self):
-        s = HTTPStorage()
-        with self.assertRaisesMessage(NotImplementedError, "Unable to save to web locations"):
-            s.save('//gitea.tfconsulting.com.au/foo', 'Some content')
+        self.assertEqual(result.my_file.name, expected)
+        self.assertGreater(result.my_file.size, 0)
 
-    def test_open(self):
-        s = HTTPStorage()
-        with s.open('//gitea.tfconsulting.com.au', 'rb') as f:
-            data = f.read()
+        self.assertEqual(os.listdir(os.path.join(self.test_dir.name, 'one')), [expected])
 
-        self.assertTrue(len(data) > 4000)
+"""

byostorage/tests/test_http_storage.py

@@ -0,0 +1,29 @@
+from django.test import TestCase
+
+from byostorage.http import HTTPStorage
+
+class HTTPStorageTestCase(TestCase):
+
+    def test_url(self):
+        s = HTTPStorage()
+
+        self.assertEqual(s.url('//google.com'), 'https://google.com')
+
+    def test_exists(self):
+        s = HTTPStorage()
+
+        self.assertTrue(s.exists('//gitea.tfconsulting.com.au/tris'))
+        self.assertFalse(s.exists('//gitea.tfconsulting.com.au/foo.txt'))
+
+    def test_save(self):
+        return # already caught as suspicious activity
+        s = HTTPStorage()
+        with self.assertRaisesMessage(NotImplementedError, "Unable to save to web locations"):
+            s.save('//gitea.tfconsulting.com.au/foo', 'Some content')
+
+    def test_open(self):
+        s = HTTPStorage()
+        with s.open('//gitea.tfconsulting.com.au', 'rb') as f:
+            data = f.read()
+
+        self.assertTrue(len(data) > 4000)

byostorage/tests/test_multi_storage.py

@@ -0,0 +1,44 @@
+from django.test import TestCase
+from byostorage.multi import MultiStorage
+from django.core.files.storage import FileSystemStorage
+
+import os.path
+
+class MultiStorageTestCase(TestCase):
+
+    def test_storage_selection(self):
+        ms = MultiStorage({'one': {'location': 'storage/one'}, 'two': {}})
+        self.assertEqual(str(ms), "<MultiStorage: 2 storages>")
+
+        # use the default if nothing set
+        default = ms.get_storage('foo')
+        self.assertIsInstance(default, FileSystemStorage)
+        self.assertEqual(default.base_location, 'media')
+
+        # get first instance
+        one = ms.get_storage('one')
+        self.assertEqual(one.base_location, 'storage/one')
+
+    def test_storage_proxies(self):
+        ms = MultiStorage({
+            'one': {'location': 'storage/one', 'base_url': 'http://one'},
+            'two': {'location': 'storage/two', 'base_url': 'http://two'}
+        })
+
+
+        self.assertEqual(ms.url("one:foo.txt"), 'http://one/foo.txt')
+        self.assertEqual(ms.url("two:foo.txt"), 'http://two/foo.txt')
+
+    def test_with_http(self):
+        ms = MultiStorage({
+            'https': {'storage': 'byostorage.http.HTTPStorage', 'protocol': 'https'}
+        })
+
+        self.assertEqual(ms.url('https://google.com'), 'https://google.com')
+
+    def test_local_methods(self):
+        ms = MultiStorage({
+            'local': {'storage': 'django.core.files.storage.FileSystemStorage', 'location': 'local_store'}
+        })
+
+        self.assertEqual(ms.path('local:foo/bar.txt'), os.path.realpath('local_store/foo/bar.txt'))

byostorage/user.py

@@ -6,7 +6,6 @@ from .multi import MultiStorage
 class BYOStorage(MultiStorage):
     ''' Database driven Bring-Your-Own-Storage
 
-    Multiple storages can 
     '''
 
     def __init__(self, config=None):

setup.cfg

@@ -11,9 +11,8 @@ classifiers =
     Development Status :: 1 - Development
     Environment :: Web Environment
     Framework :: Django
-    Framework :: Django :: 2.2
-    Framework :: Django :: 3.0
-    Framework :: Django :: 3.1
+    Framework :: Django :: 4.2
+    Framework :: Django :: 5.0
     Intended Audience :: Developers
     License :: OSI Approved :: BSD License
     Operating System :: OS Independent
@@ -29,7 +28,7 @@ classifiers =
 [options]
 python_requires = >=3.5
 install_requires =
-    Django >= 2.2
+    Django >= 4.2
 packages =
     byostorage
     byostorage.migrations