Merged in jmchilton/galaxy-central-fork-1 (pull request #342)

Refactor History Contents to Support Multiple Types

test/base/interactor.py

@@ -461,16 +461,21 @@ except ImportError:
     def post_request( url, data, files={} ):
         return __multipart_request( url, data, files, verb="POST" )
 
-    def put_request( url ):
-        return __urllib_request( url, 'PUT' )
+    def put_request( url, data=None ):
+        if isinstance( data, dict ):
+            assert False, "This test will fail, Galaxy's webob dependency does not parse out urlencoded PUT/PATCH entity data, API will receive empty payload."
+        return __urllib_request( url, 'PUT', json_str=data )
 
     def delete_request( url ):
         return __urllib_request( url, 'DELETE' )
 
-    def __urllib_request( url, verb ):
+    def __urllib_request( url, verb, json_str=None ):
         opener = urllib2.build_opener(urllib2.HTTPHandler)
         request = urllib2.Request(url)
         request.get_method = lambda: verb
+        if json_str:
+            request.add_header( "content-type", "application/json" )
+            request.add_data(json_str)
         try:
             response = opener.open(request)
             return RequestsLikeResponse( response.read(), status_code=response.getcode() )

test/functional/api/helpers.py

@@ -1,6 +1,6 @@
 import time
-from json import dumps
-from json import loads
+import json
+import StringIO
 from pkg_resources import resource_string
 
 # Simple workflow that takes an input and call cat wrapper on it.
@@ -10,33 +10,46 @@ workflow_str = resource_string( __name__, "test_workflow_1.ga" )
 workflow_random_x2_str = resource_string( __name__, "test_workflow_2.ga" )
 
 
+# Deprecated mixin, use dataset populator instead.
+# TODO: Rework existing tests to target DatasetPopulator in a setup method instead.
 class TestsDatasets:
 
     def _new_dataset( self, history_id, content='TestData123', **kwds ):
-        payload = self._upload_payload( history_id, content, **kwds )
-        run_response = self._post( "tools", data=payload )
-        self._assert_status_code_is( run_response, 200 )
-        return run_response.json()["outputs"][0]
+        return DatasetPopulator( self.galaxy_interactor ).new_dataset( history_id, content=content, **kwds)
 
     def _wait_for_history( self, history_id, assert_ok=False ):
-        while True:
-            history_details_response = self._get( "histories/%s" % history_id )
-            self._assert_status_code_is( history_details_response, 200 )
-            history_state = history_details_response.json()[ "state" ]
-            if history_state not in [ "running", "queued" ]:
-                break
-            time.sleep( .1 )
-        if assert_ok:
-            self.assertEquals( history_state, 'ok' )
+        return DatasetPopulator( self.galaxy_interactor ).wait_for_history( history_id, assert_ok=assert_ok )
 
     def _new_history( self, **kwds ):
+        return DatasetPopulator( self.galaxy_interactor ).new_history( **kwds )
+
+    def _upload_payload( self, history_id, content, **kwds ):
+        return DatasetPopulator( self.galaxy_interactor ).upload_payload( history_id, content, **kwds )
+
+    def _run_tool_payload( self, tool_id, inputs, history_id, **kwds ):
+        return DatasetPopulator( self.galaxy_interactor ).run_tool_payload( tool_id, inputs, history_id, **kwds )
+
+
+class DatasetPopulator( object ):
+
+    def __init__( self, galaxy_interactor ):
+        self.galaxy_interactor = galaxy_interactor
+
+    def new_dataset( self, history_id, content='TestData123', **kwds ):
+        payload = self.upload_payload( history_id, content, **kwds )
+        run_response = self.galaxy_interactor.post( "tools", data=payload )
+        return run_response.json()["outputs"][0]
+
+    def wait_for_history( self, history_id, assert_ok=False ):
+        wait_on_state( lambda: self.galaxy_interactor.get( "histories/%s" % history_id ), assert_ok=assert_ok )
+
+    def new_history( self, **kwds ):
         name = kwds.get( "name", "API Test History" )
-        create_history_response = self._post( "histories", data=dict( name=name ) )
-        self._assert_status_code_is( create_history_response, 200 )
+        create_history_response = self.galaxy_interactor.post( "histories", data=dict( name=name ) )
         history_id = create_history_response.json()[ "id" ]
         return history_id
 
-    def _upload_payload( self, history_id, content, **kwds ):
+    def upload_payload( self, history_id, content, **kwds ):
         name = kwds.get( "name", "Test Dataset" )
         dbkey = kwds.get( "dbkey", "?" )
         file_type = kwds.get( "file_type", 'txt' )
@@ -50,17 +63,17 @@ class TestsDatasets:
             upload_params[ "files_0|to_posix_lines"] = kwds[ "to_posix_lines" ]
         if "space_to_tab" in kwds:
             upload_params[ "files_0|space_to_tab" ] = kwds[ "space_to_tab" ]
-        return self._run_tool_payload(
+        return self.run_tool_payload(
             tool_id='upload1',
             inputs=upload_params,
             history_id=history_id,
             upload_type='upload_dataset'
         )
 
-    def _run_tool_payload( self, tool_id, inputs, history_id, **kwds ):
+    def run_tool_payload( self, tool_id, inputs, history_id, **kwds ):
         return dict(
             tool_id=tool_id,
-            inputs=dumps(inputs),
+            inputs=json.dumps(inputs),
             history_id=history_id,
             **kwds
         )
@@ -73,7 +86,7 @@ class WorkflowPopulator( object ):
         self.api_test_case = api_test_case
 
     def load_workflow( self, name, content=workflow_str, add_pja=False ):
-        workflow = loads( content )
+        workflow = json.loads( content )
         workflow[ "name" ] = name
         if add_pja:
             tool_step = workflow[ "steps" ][ "2" ]
@@ -93,9 +106,99 @@ class WorkflowPopulator( object ):
 
     def create_workflow( self, workflow, **create_kwds ):
         data = dict(
-            workflow=dumps( workflow ),
+            workflow=json.dumps( workflow ),
             **create_kwds
         )
         upload_response = self.api_test_case._post( "workflows/upload", data=data )
         uploaded_workflow_id = upload_response.json()[ "id" ]
         return uploaded_workflow_id
+
+
+class LibraryPopulator( object ):
+
+    def __init__( self, api_test_case ):
+        self.api_test_case = api_test_case
+        self.galaxy_interactor = api_test_case.galaxy_interactor
+
+    def new_private_library( self, name ):
+        library = self.new_library( name )
+        library_id = library[ "id" ]
+
+        role_id = self.user_private_role_id()
+        self.set_permissions( library_id, role_id )
+        return library
+
+    def new_library( self, name ):
+        data = dict( name=name )
+        create_response = self.galaxy_interactor.post( "libraries", data=data, admin=True )
+        return create_response.json()
+
+    def set_permissions( self, library_id, role_id=None ):
+        if role_id:
+            perm_list = json.dumps( role_id )
+        else:
+            perm_list = json.dumps( [] )
+
+        permissions = dict(
+            LIBRARY_ACCESS_in=perm_list,
+            LIBRARY_MODIFY_in=perm_list,
+            LIBRARY_ADD_in=perm_list,
+            LIBRARY_MANAGE_in=perm_list,
+        )
+        self.galaxy_interactor.post( "libraries/%s/permissions" % library_id, data=permissions, admin=True )
+
+    def user_email( self ):
+        users_response = self.galaxy_interactor.get( "users" )
+        users = users_response.json()
+        assert len( users ) == 1
+        return users[ 0 ][ "email" ]
+
+    def user_private_role_id( self ):
+        user_email = self.user_email()
+        roles_response = self.api_test_case.galaxy_interactor.get( "roles", admin=True )
+        users_roles = [ r for r in roles_response.json() if r[ "name" ] == user_email ]
+        assert len( users_roles ) == 1
+        return users_roles[ 0 ][ "id" ]
+
+    def create_dataset_request( self, library, **kwds ):
+        create_data = {
+            "folder_id": kwds.get( "folder_id", library[ "root_folder_id" ] ),
+            "create_type": "file",
+            "files_0|NAME": kwds.get( "name", "NewFile" ),
+            "upload_option": kwds.get( "upload_option", "upload_file" ),
+            "file_type": kwds.get( "file_type", "auto" ),
+            "db_key": kwds.get( "db_key", "?" ),
+        }
+        files = {
+            "files_0|file_data": kwds.get( "file", StringIO.StringIO( kwds.get( "contents", "TestData" ) ) ),
+        }
+        return create_data, files
+
+    def new_library_dataset( self, name, **create_dataset_kwds ):
+        library = self.new_private_library( name )
+        payload, files = self.create_dataset_request( library, **create_dataset_kwds )
+        url_rel = "libraries/%s/contents" % ( library[ "id" ] )
+        dataset = self.api_test_case.galaxy_interactor.post( url_rel, payload, files=files ).json()[0]
+
+        def show():
+            return self.api_test_case.galaxy_interactor.get( "libraries/%s/contents/%s" % ( library[ "id" ], dataset[ "id" ] ) )
+
+        wait_on_state(show)
+        return show().json()
+
+
+def wait_on_state( state_func, assert_ok=False, timeout=5 ):
+    delta = .1
+    iteration = 0
+    while True:
+        if (delta * iteration) > timeout:
+            assert False, "Timed out waiting on state."
+        iteration += 1
+        response = state_func()
+        assert response.status_code == 200, "Failed to fetch state update while waiting."
+        state = response.json()[ "state" ]
+        if state not in [ "running", "queued", "new" ]:
+            break
+        time.sleep( delta )
+    if assert_ok:
+        assert state == "ok", "Final state - %s - not okay." % state

test/functional/api/test_history_contents.py

@@ -0,0 +1,111 @@
+from base import api
+import json
+
+from .helpers import TestsDatasets
+from .helpers import LibraryPopulator
+from base.interactor import (
+    put_request,
+    delete_request,
+)
+
+
+# TODO: Test anonymous access.
+class HistoryContentsApiTestCase( api.ApiTestCase, TestsDatasets ):
+
+    def setUp( self ):
+        super( HistoryContentsApiTestCase, self ).setUp()
+        self.history_id = self._new_history()
+
+    def test_index_hda_summary( self ):
+        hda1 = self._new_dataset( self.history_id )
+        contents_response = self._get( "histories/%s/contents" % self.history_id )
+        hda_summary = self.__check_for_hda( contents_response, hda1 )
+        assert "display_types" not in hda_summary  # Quick summary, not full details
+
+    def test_index_hda_all_details( self ):
+        hda1 = self._new_dataset( self.history_id )
+        contents_response = self._get( "histories/%s/contents?details=all" % self.history_id )
+        hda_details = self.__check_for_hda( contents_response, hda1 )
+        self.__assert_hda_has_full_details( hda_details )
+
+    def test_index_hda_detail_by_id( self ):
+        hda1 = self._new_dataset( self.history_id )
+        contents_response = self._get( "histories/%s/contents?details=%s" % ( self.history_id, hda1[ "id" ] ) )
+        hda_details = self.__check_for_hda( contents_response, hda1 )
+        self.__assert_hda_has_full_details( hda_details )
+
+    def test_show_hda( self ):
+        hda1 = self._new_dataset( self.history_id )
+        show_response = self.__show( hda1 )
+        self._assert_status_code_is( show_response, 200 )
+        self.__assert_matches_hda( hda1, show_response.json() )
+
+    def test_hda_copy( self ):
+        hda1 = self._new_dataset( self.history_id )
+        create_data = dict(
+            source='hda',
+            content=hda1[ "id" ],
+        )
+        second_history_id = self._new_history()
+        assert self.__count_contents( second_history_id ) == 0
+        create_response = self._post( "histories/%s/contents" % second_history_id, create_data )
+        self._assert_status_code_is( create_response, 200 )
+        assert self.__count_contents( second_history_id ) == 1
+
+    def test_library_copy( self ):
+        ld = LibraryPopulator( self ).new_library_dataset( "lda_test_library" )
+        create_data = dict(
+            source='library',
+            content=ld[ "id" ],
+        )
+        assert self.__count_contents( self.history_id ) == 0
+        create_response = self._post( "histories/%s/contents" % self.history_id, create_data )
+        self._assert_status_code_is( create_response, 200 )
+        assert self.__count_contents( self.history_id ) == 1
+
+    def test_update( self ):
+        hda1 = self._new_dataset( self.history_id )
+        self._wait_for_history( self.history_id )
+        assert str( hda1[ "deleted" ] ).lower() == "false"
+        update_url = self._api_url( "histories/%s/contents/%s" % ( self.history_id, hda1[ "id" ] ), use_key=True )
+        # Awkward json.dumps required here because of https://trello.com/c/CQwmCeG6
+        body = json.dumps( dict( deleted=True ) )
+        update_response = put_request( update_url, data=body )
+        self._assert_status_code_is( update_response, 200 )
+        show_response = self.__show( hda1 )
+        assert str( show_response.json()[ "deleted" ] ).lower() == "true"
+
+    def test_delete( self ):
+        hda1 = self._new_dataset( self.history_id )
+        self._wait_for_history( self.history_id )
+        assert str( self.__show( hda1 ).json()[ "deleted" ] ).lower() == "false"
+        url = self._api_url( "histories/%s/contents/%s" % ( self.history_id, hda1["id" ] ), use_key=True )
+        delete_response = delete_request( url )
+        assert delete_response.status_code < 300  # Something in the 200s :).
+        assert str( self.__show( hda1 ).json()[ "deleted" ] ).lower() == "true"
+
+    def __show( self, hda ):
+        show_response = self._get( "histories/%s/contents/%s" % ( self.history_id, hda[ "id" ] ) )
+        return show_response
+
+    def __count_contents( self, history_id=None, **kwds ):
+        if history_id == None:
+            history_id = self.history_id
+        contents_response = self._get( "histories/%s/contents" % history_id, kwds )
+        return len( contents_response.json() )
+
+    def __assert_hda_has_full_details( self, hda_details ):
+        self._assert_has_keys( hda_details, "display_types", "display_apps" )
+
+    def __check_for_hda( self, contents_response, hda ):
+        self._assert_status_code_is( contents_response, 200 )
+        contents = contents_response.json()
+        assert len( contents ) == 1
+        hda_summary = contents[ 0 ]
+        self.__assert_matches_hda( hda, hda_summary )
+        return hda_summary
+
+    def __assert_matches_hda( self, input_hda, query_hda ):
+        self._assert_has_keys( query_hda, "id", "name" )
+        assert input_hda[ "name" ] == query_hda[ "name" ]
+        assert input_hda[ "id" ] == query_hda[ "id" ]

test/functional/api/test_libraries.py

@@ -0,0 +1,54 @@
+from base import api
+from .helpers import TestsDatasets
+from .helpers import LibraryPopulator
+from .helpers import wait_on_state
+
+
+class LibrariesApiTestCase( api.ApiTestCase, TestsDatasets ):
+
+    def setUp( self ):
+        super( LibrariesApiTestCase, self ).setUp()
+        self.library_populator = LibraryPopulator( self )
+
+    def test_create( self ):
+        data = dict( name="CreateTestLibrary" )
+        create_response = self._post( "libraries", data=data, admin=True )
+        self._assert_status_code_is( create_response, 200 )
+        library = create_response.json()
+        self._assert_has_keys( library, "name" )
+        assert library[ "name" ] == "CreateTestLibrary"
+
+    def test_create_private_library_permissions( self ):
+        library = self.library_populator.new_library( "PermissionTestLibrary" )
+        library_id = library[ "id" ]
+
+        role_id = self.library_populator.user_private_role_id()
+        self.library_populator.set_permissions( library_id, role_id )
+        create_response = self._create_folder( library )
+        self._assert_status_code_is( create_response, 200 )
+
+    def test_create_dataset( self ):
+        library = self.library_populator.new_private_library( "ForCreateDatasets" )
+        payload, files = self.library_populator.create_dataset_request( library, file_type="txt", contents="create_test" )
+        create_response = self._post( "libraries/%s/contents" % library[ "id" ], payload, files=files )
+        self._assert_status_code_is( create_response, 200 )
+        library_datasets = create_response.json()
+        assert len( library_datasets ) == 1
+        library_dataset = library_datasets[ 0 ]
+
+        def show():
+            return self._get( "libraries/%s/contents/%s" % ( library[ "id" ], library_dataset[ "id" ] ) )
+
+        wait_on_state( show, assert_ok=True )
+        library_dataset = show().json()
+        self._assert_has_keys( library_dataset, "peek", "data_type" )
+        assert library_dataset[ "peek" ].find("create_test") >= 0
+        assert library_dataset[ "data_type" ] == "txt"
+
+    def _create_folder( self, library ):
+        create_data = dict(
+            folder_id=library[ "root_folder_id" ],
+            create_type="folder",
+            name="New Folder",
+        )
+        return self._post( "libraries/%s/contents" % library[ "id" ], data=create_data )