Basic web sync working

polyphonic/interface/static/interface/js/interface.js

@@ -26,4 +26,31 @@ document.addEventListener('DOMContentLoaded', () => {
       }
   }
 
-});
+});
+
+  async function background_api(el) {
+    console.log(el.dataset["api"]);
+    let text = el.innerHTML;
+    el.disabled = true;
+    el.innerHTML = "Running..."
+    const response = await fetch(el.dataset.api);
+
+    console.log(response);
+      
+    el.innerHTML = text;
+
+    if(!response.ok) {
+      el.disabled = false;
+      throw new Error(`Response: ${response.status}`);
+    }
+
+    if (el.dataset.success) {
+      text = el.dataset.success;
+    }
+
+    el.innerHTML = text;
+    const data = await response.json();
+    
+    console.log(data);
+
+  }

polyphonic/library/gdrive/__init__.py

@@ -14,6 +14,13 @@ def sync_work(work: Work):
 
     logger.info("Syncing '%s' from %r", work.name, folder_id)
 
+    results = {
+        "folder_id": folder_id,
+        "added": [],
+        "skipped": [],
+        "missing": [],
+    }
+
     storage = UserStorage.objects.get(name="gdrive").instance()
 
     existing = set(
@@ -27,6 +34,8 @@ def sync_work(work: Work):
     _, files = storage.listdir(folder_id)
     logger.debug("Remote files: %r", files)
 
+    results["files"] = len(files)
+
     for file in files:
         if file.id in existing:
             logger.debug("%30s: Skipping existing (%s)", file.name, file.id)
@@ -35,22 +44,29 @@ def sync_work(work: Work):
 
         if not file.name.lower().endswith(".pdf"):
             logger.debug("%40s: Not a PDF", file.name)
+            results["skipped"].append({"file": file.name, "file_id": file.id})
             continue
 
-        logger.info("%40s: Adding", file.name)
+        results["added"].append({"file": file.name, "file_id": file.id})
         doc = work.docs.create(upload=f"gdrive:{file}", doctype=Document.DOCTYPE_PDF)
         doc.auto_tag()
 
     for uri in existing:
+        results["missing"].append({"uri": uri})
         logger.warning("Local entry not in folder: %s", uri)
 
+    return results
+
 
 def sync_partial_collection(collection: Collection, sync_existing: bool = True):
-
     works = Work.objects.filter(collection=collection, meta_info__name="folderid")
 
+    result = []
+
     for work in works:
-        sync_work(work)
+        result.append(sync_work(work))
+
+    return result
 
 
 def sync_collection(collection: Collection, sync_existing: bool = False):
@@ -69,6 +85,12 @@ def sync_collection(collection: Collection, sync_existing: bool = False):
     storage = collection.storage.instance()
     folders, _ = storage.listdir(folder)
 
+    results = {
+        "folders": len(folders),
+        "added": [],
+        "missing": [],
+    }
+
     for folder in folders:
         if folder[0] == "_":
             continue
@@ -84,7 +106,11 @@ def sync_collection(collection: Collection, sync_existing: bool = False):
         work = Work(name=folder.name, collection=collection)
         work.save()
         work.meta_info.create(name="folderid", value=folder.id)
+        results["added"].append({"work": work.pk, "folder_id": folder.id})
         sync_work(work)
 
-    for folderid, work in existing:
+    for folderid, work in existing.items():
+        results["missing"].append({"work": work, "folder_id": folderid})
         logger.warning("Folder for work %d no longer in drive (%s)", work, folderid)
+
+    return results

polyphonic/library/gdrive/storage.py

@@ -69,20 +69,26 @@ class GDriveLinkStorage(Storage):
         if path == "":
             return [], []
 
-        folder = self.parse_resource(path)
-        url = f"{FILES_API}?q='{folder.id}'+in+parents&key={self.api_key}"
-        data = self.get_json(url, folder)
         files = []
         folders = []
-        for x in data["files"]:
-            if x["mimeType"] == "application/vnd.google-apps.folder":
-                # folders.append(f"{x['id']}/{x['name']}")
-                folders.append(DriveObject(x["id"], x.get("resourceKey"), x["name"]))
-            else:
-                # files.append(f"{x['id']}/{x['name']}")
-                files.append(DriveObject(x["id"], x.get("resourceKey"), x["name"]))
 
-        return folders, files
+        folder = self.parse_resource(path)
+        url = f"{FILES_API}?q='{folder.id}'+in+parents&key={self.api_key}"
+
+        data = self.get_json(url, folder)
+        while True:
+            for x in data["files"]:
+                if x["mimeType"] == "application/vnd.google-apps.folder":
+                    folders.append(
+                        DriveObject(x["id"], x.get("resourceKey"), x["name"])
+                    )
+                else:
+                    files.append(DriveObject(x["id"], x.get("resourceKey"), x["name"]))
+
+            token = data.get("nextPageToken")
+            if token is None:
+                return folders, files
+            data = self.get_json(f"{url}&pageToken={token}", folder)
 
     def get_meta(self, name):
         file_resource = self.parse_resource(name)

polyphonic/library/templates/library/gdrive.html

@@ -1,9 +1,17 @@
 {% extends "interface/project_base.html" %}
 
 {% block admin %}
+
+{% if meta.folderid %}
+<button class="button is-link" data-api="{% url 'work_sync' work=object.pk %}" data-success="Synced" onclick="background_api(this)">
+                         Sync with Drive
+  </button>
+{% endif %}
+
 <a href="{% url 'work_detail' collection=collection.pk pk=object.pk %}" class="button is-link is-light">
   <span>Back to work</span>
 </a>
+
 {% endblock %}
 
 {% block page %}
@@ -14,7 +22,9 @@
 <div class="m-3">
 <p>
     {% if meta.folderid %}
-This work is currently linked to <b>{{ meta.folderid }}</b>.  Pasting a new folder link will overwrite this.
+This work is currently linked to folder <span class="tag is-success">{{ meta.folderid }}</span><br/><em>Pasting a new folder link will overwrite this.</em>
+
+
     {% else %}
 There is currently no shared drive folder linked to this work - paste one here to enable syncing.
     {% endif %}
@@ -38,4 +48,6 @@ There is currently no shared drive folder linked to this work - paste one here t
   {% csrf_token %}
 </form>
 </div>
+
 {% endblock %}
+

polyphonic/library/templates/library/work_list.html

@@ -2,12 +2,20 @@
 {% load url_tools %}
 
 {% block admin %}
+
 {% if collection %}
+
+
+<button class="button is-link" data-api="{% url 'collection_sync' collection=collection.pk %}" data-success="Synced" onclick="background_api(this)">
+  Sync Collection
+</button>
+
 <a href="{% url 'work_add' collection.pk %}" class="button is-link">
     <span class="icon"><i class="fas fa-plus-circle"></i></span>
     <span>Add a work</span>
 </a>
 {% endif %}
+
 {% endblock %}
 
 {% block page %}

polyphonic/library/urls.py

@@ -126,4 +126,14 @@ urlpatterns = [
         api.CollectionImportView.as_view(),
         name="collection_import",
     ),
+    path(
+        "api/collections/<int:collection>/sync",
+        api.CollectionSyncView.as_view(),
+        name="collection_sync",
+    ),
+    path(
+        "api/works/<int:work>/sync",
+        api.WorkSyncView.as_view(),
+        name="work_sync",
+    ),
 ]

polyphonic/library/views/api.py

@@ -3,9 +3,12 @@ from polyphonic.interface.views import AuthorizedResourceMixin
 from rest_framework import serializers
 from rest_framework.exceptions import APIException
 from rest_framework import generics
+from rest_framework.views import APIView
+from rest_framework.response import Response
 
 from polyphonic.library.models import Collection, Work, Document, Section, WorkMeta
 
+from polyphonic.library.gdrive import sync_collection, sync_work
 
 import requests
 import urllib
@@ -208,3 +211,25 @@ class CollectionImportView(AuthorizedResourceMixin, generics.CreateAPIView):
 
     def perform_create(self, serializer):
         serializer.save(collection_id=self.kwargs["pk"])
+
+
+class WorkSyncView(AuthorizedResourceMixin, APIView):
+    admin_required = True
+
+    def get(self, request, work, format=None):
+        obj = Work.objects.get(pk=work)
+
+        result = sync_work(obj)
+
+        return Response(result)
+
+
+class CollectionSyncView(AuthorizedResourceMixin, APIView):
+    admin_required = True
+
+    def get(self, request, collection, format=None):
+        obj = Collection.objects.get(pk=collection)
+
+        result = sync_collection(obj)
+
+        return Response(result)