[GRDM-62817] Resolve storage providers from project API

.github/workflows/unit-tests.yml

@@ -1,6 +1,7 @@
 name: Unit Tests
 
 on:
+  pull_request:
   push:
     branches-ignore:
       - '[0-9]*'

docs/cli-usage.rst

@@ -42,6 +42,16 @@ Below some examples on how to use the command-line program:
 
 If you're using python 3+, you can also use the aliases `ls` in place of `list`, and `rm` in place of `remove`.
 
+Remote paths can start with a provider identifier, such as
+``s3compatsigv4/folder/file.txt``. Providers are discovered from the project's
+storage API. If the first path component names a provider the server offers
+but the project has not connected, the command fails instead of writing into
+``osfstorage``. Addons in the ``other`` category, such as ``binderhub``, do
+not hold files and are not treated as providers. If it matches no provider
+at all, the path refers to the default ``osfstorage``.
+To access a folder whose name matches a provider, explicitly prefix the path
+with ``osfstorage/``.
+
 
 If the project is private you will need to provide authentication
 details. You can set the ``OSF_TOKEN`` environment

osfclient/api.py

@@ -1,4 +1,5 @@
 from .exceptions import OSFException
+from .models import Addon
 from .models import OSFCore
 from .models import Project
 
@@ -46,6 +47,16 @@ async def project(self, project_id):
             }
         }, self.session)
 
+    @property
+    async def addons(self):
+        """Iterate over all addons available on the server."""
+        url = self._build_url('addons')
+        while url:
+            response = self._json(await self._get(url), 200)
+            for addon in response['data']:
+                yield Addon(addon, self.session)
+            url = response['links']['next']
+
     @property
     def token(self):
         if 'Authorization' not in self.session.headers:

osfclient/cli.py

@@ -184,8 +184,9 @@ async def fetch(args):
     """Fetch an individual file from a project.
 
     The first part of the remote path is interpreted as the name of the
-    storage provider. If there is no match the default (osfstorage) is
-    used.
+    connected storage provider. A provider that exists on the server but
+    is not connected to the project is an error. If there is no match,
+    the default (osfstorage) is used.
 
     The local path defaults to the name of the remote file.
 
@@ -195,7 +196,9 @@ async def fetch(args):
     If args.force is False but args.update is True, overwrite an existing local
     file only if local and remote files differ.
     """
-    storage, remote_path = split_storage(args.remote)
+    osf = _setup_osf(args)
+    project = await osf.project(args.project)
+    store, remote_path = await split_storage(args.remote, osf, project)
 
     local_path = args.local
     if local_path is None:
@@ -209,10 +212,6 @@ async def fetch(args):
     if directory:
         makedirs(directory, exist_ok=True)
 
-    osf = _setup_osf(args)
-    project = await osf.project(args.project)
-
-    store = await project.storage(storage)
     # only fetching one file so we are done
     file_ = await find_by_path(store, remote_path)
     if file_ is None or is_folder(file_):
@@ -279,8 +278,9 @@ async def upload(args):
     """Upload a new file to an existing project.
 
     The first part of the remote path is interpreted as the name of the
-    storage provider. If there is no match the default (osfstorage) is
-    used.
+    connected storage provider. A provider that exists on the server but
+    is not connected to the project is an error. If there is no match,
+    the default (osfstorage) is used.
 
     If the project is private you need to specify a username or token.
 
@@ -300,9 +300,8 @@ async def upload(args):
         sys.exit('To upload a file you need to provide a token.')
 
     project = await osf.project(args.project)
-    storage, remote_path = split_storage(args.destination)
+    store, remote_path = await split_storage(args.destination, osf, project)
 
-    store = await project.storage(storage)
     if args.recursive:
         if not os.path.isdir(args.source):
             raise RuntimeError("Expected source ({}) to be a directory when "
@@ -333,18 +332,18 @@ async def makefolder(args):
     """Create a new folder in an existing project.
 
     The first part of the remote path is interpreted as the name of the
-    storage provider. If there is no match the default (osfstorage) is
-    used.
+    connected storage provider. A provider that exists on the server but
+    is not connected to the project is an error. If there is no match,
+    the default (osfstorage) is used.
     """
     osf = _setup_osf(args)
     if not osf.has_auth:
         sys.exit('To create a folder you need to provide a token.')
 
     project = await osf.project(args.project)
 
-    storage, remote_path = split_storage(args.target)
+    store, remote_path = await split_storage(args.target, osf, project)
 
-    store = await project.storage(storage)
     f = await find_ancestral_folder(store, remote_path)
     if f is None:
         parent = store
@@ -362,18 +361,18 @@ async def remove(args):
     """Remove a file from the project's storage.
 
     The first part of the remote path is interpreted as the name of the
-    storage provider. If there is no match the default (osfstorage) is
-    used.
+    connected storage provider. A provider that exists on the server but
+    is not connected to the project is an error. If there is no match,
+    the default (osfstorage) is used.
     """
     osf = _setup_osf(args)
     if not osf.has_auth:
         sys.exit('To remove a file you need to provide a token.')
 
     project = await osf.project(args.project)
 
-    storage, remote_path = split_storage(args.target)
+    store, remote_path = await split_storage(args.target, osf, project)
 
-    store = await project.storage(storage)
     f = await find_by_path(store, remote_path)
     if f is None:
         sys.exit('No files found to remove.')
@@ -385,16 +384,19 @@ async def move(args):
     """Move a file to specified location on the project's storage.
 
     The first part of the paths is interpreted as the name of the
-    storage provider. If there is no match the default (osfstorage) is
-    used.
+    connected storage provider. A provider that exists on the server but
+    is not connected to the project is an error. If there is no match,
+    the default (osfstorage) is used.
     """
     osf = _setup_osf(args)
     if not osf.has_auth:
         sys.exit('To move a file you need to provide a token.')
 
     project = await osf.project(args.project)
 
-    target_storage, target_path = split_storage(args.target, normalize=False)
+    target_store, target_path = await split_storage(
+        args.target, osf, project, normalize=False)
+    target_storage = target_store.provider
 
     if target_path.endswith('/'):
         target_folder_path = target_path[:-1]
@@ -409,16 +411,14 @@ async def move(args):
     else:
         target_folder_path = None
         target_filename = target_path
-    target_store = await project.storage(target_storage)
     if target_folder_path is None:
         target_folder = target_store
     else:
         target_folder = await _ensure_folder(target_store, target_folder_path)
 
     # Move a file
-    storage, remote_path = split_storage(args.source)
+    store, remote_path = await split_storage(args.source, osf, project)
 
-    store = await project.storage(storage)
     f = await find_by_path(store, remote_path)
     if f is None:
         sys.exit('No files found to move.')

osfclient/models/__init__.py

@@ -3,6 +3,7 @@
 Users should not have to instantiate classes from here, instead they should
 use `osfclient.OSF()` to access the OSF.
 """
+from .addon import Addon
 from .core import OSFCore
 from .file import File
 from .file import Folder

osfclient/models/addon.py

@@ -0,0 +1,11 @@
+from .core import OSFCore
+
+
+class Addon(OSFCore):
+    def _update_attributes(self, addon):
+        self.id = self._get_attribute(addon, 'id')
+        self.name = self._get_attribute(addon, 'attributes', 'name')
+        self.categories = self._get_attribute(addon, 'attributes', 'categories')
+
+    def __str__(self):
+        return '<Addon [{0}]>'.format(self.id)

osfclient/models/project.py

@@ -38,20 +38,19 @@ def __str__(self):
 
     async def storage(self, provider='osfstorage'):
         """Return storage `provider`."""
-        stores = self._json(await self._get(self._storages_url), 200)
-        stores = stores['data']
-        for store in stores:
-            provides = self._get_attribute(store, 'attributes', 'provider')
-            if provides == provider:
-                return Storage(store, self.session)
+        async for store in self.storages:
+            if store.provider == provider:
+                return store
 
         raise RuntimeError("Project has no storage "
                            "provider '{}'".format(provider))
 
     @property
     async def storages(self):
         """Iterate over all storages for this projects."""
-        stores = self._json(await self._get(self._storages_url), 200)
-        stores = stores['data']
-        for store in stores:
-            yield Storage(store, self.session)
+        url = self._storages_url
+        while url:
+            response = self._json(await self._get(url), 200)
+            for store in response['data']:
+                yield Storage(store, self.session)
+            url = response['links']['next']

osfclient/tests/fake_responses.py

@@ -53,6 +53,26 @@ def storage_node(project_id, storages=['osfstorage']):
                                'n_storages': len(used_storages)})
 
 
+# Use this to fake a response when asking for the server's addons
+# e.g. osf.addons
+def addons(ids, categories=['storage']):
+    return {
+        'data': [{
+            'id': addon_id,
+            'type': 'addon',
+            'attributes': {'name': addon_id, 'categories': categories},
+            'links': {},
+        } for addon_id in ids],
+        'links': {
+            'first': None,
+            'last': None,
+            'prev': None,
+            'next': None,
+            'meta': {'total': len(ids), 'per_page': 1000},
+        },
+    }
+
+
 def _folder(osf_id, name, storage='osfstorage'):
     template = """{
         "relationships": {

osfclient/tests/mocks.py

@@ -1,4 +1,5 @@
 import asyncio
+from types import SimpleNamespace
 from mock import MagicMock, PropertyMock, AsyncMock
 from ..utils import norm_remote_path
 import copy
@@ -58,7 +59,7 @@ def MockStorage(name):
         MockFolder('/a',folders=a_folders),
         MockFolder('/b',folders=b_folders),
         MockFolder('/c',folders=c_folders)]
-    mock = MagicMock(name='Storage-%s' % name,
+    mock = MagicMock(name='Storage-%s' % name, provider=name,
                      folders=AsyncIterator(folders),
                      children=AsyncIterator(folders))
     mock.create_file = MagicMock(return_value=FutureWrapper())
@@ -99,16 +100,26 @@ def MockStream(path, mode, size=1024):
 
 
 def MockProject(name):
-    mock = MagicMock(name='Project-%s' % name,
-                     storages=AsyncIterator([MockStorage('osfstorage'), MockStorage('gh')]))
+    default_store = MockStorage('osfstorage')
+    mock = MagicMock(name='Project-%s' % name, id=name,
+                     storages=AsyncIterator([default_store, MockStorage('gh')]))
     storage = MagicMock(name='Project-%s-storage' % name,
-                        return_value=FutureMockStorage('osfstorage'))
+                        return_value=FutureWrapper(default_store))
     type(mock).storage = storage
     mock._storage_mock = storage
 
     return mock
 
 
+def MockAddon(addon_id, categories=['storage']):
+    return SimpleNamespace(id=addon_id, name=addon_id, categories=categories)
+
+
+def MockAddons(addons):
+    """Stand-in for the `OSF.addons` property."""
+    return PropertyMock(return_value=AsyncIterator(addons))
+
+
 def MockArgs(output=None, project=None,
              source=None, destination=None, local=None, remote=None,
              target=None, force=False, update=False, recursive=False,

osfclient/tests/test_api.py

@@ -7,6 +7,7 @@
 from osfclient.models import OSFCore
 from osfclient.models import Project
 
+from osfclient.tests import fake_responses
 from osfclient.tests.mocks import FakeResponse
 
 
@@ -34,3 +35,34 @@ def test_endpoint(session_set_endpoint):
 
     osf = OSF(base_url='https://api.test.osf.io/v2/')
     session_set_endpoint.assert_called_with('https://api.test.osf.io/v2/')
+
+
+@pytest.mark.asyncio
+@patch.object(OSFCore, '_get')
+async def test_addons(OSFCore_get):
+    osf = OSF(base_url='https://api.test.osf.io/v2/')
+    first_url = 'https://api.test.osf.io/v2/addons/'
+    next_url = first_url + '?page=2'
+    first = fake_responses.addons(['s3', 'github'])
+    first['links']['next'] = next_url
+    second = fake_responses.addons(['binderhub'], categories=['other'])
+    OSFCore_get.side_effect = [FakeResponse(200, first),
+                               FakeResponse(200, second)]
+
+    addons = [addon async for addon in osf.addons]
+
+    assert [addon.id for addon in addons] == ['s3', 'github', 'binderhub']
+    assert [addon.categories for addon in addons] == [
+        ['storage'], ['storage'], ['other']]
+    assert all(addon.session is osf.session for addon in addons)
+    assert OSFCore_get.call_args_list == [call(first_url), call(next_url)]
+
+
+@pytest.mark.asyncio
+@patch.object(OSFCore, '_get', return_value=FakeResponse(403, {}))
+async def test_addons_propagates_api_error(OSFCore_get):
+    osf = OSF()
+
+    with pytest.raises(RuntimeError, match='403'):
+        async for _ in osf.addons:
+            pass