Merge pull request #2664 from dannon/dev_reports

Reports: new pages and improvements
2026-09-24 16:30:27 +08:00 · 2016-07-27 09:21:33 -04:00
parent fa4085be54 8483f2ce8b
commit b74fd63fef
12 changed files with 1213 additions and 11 deletions
@@ -0,0 +1,192 @@
+import galaxy.model
+import collections
+import logging
+import sqlalchemy as sa
+
+from galaxy import util
+from galaxy.web.base.controller import BaseUIController, web
+
+from markupsafe import escape
+from sqlalchemy import and_
+
+log = logging.getLogger( __name__ )
+
+
+def int_to_octet(size):
+    try:
+        size = float(size)
+    except ValueError:
+        return "???"
+    except TypeError:
+        if size is None:
+            return "0 o"
+        return "???"
+    units = ("o", "Ko", "Mo", "Go", "To")
+    no_unit = 0
+    while (size >= 1000):
+        size /= 1000.
+        no_unit += 1
+    try:
+        return "%.2f %s" % (size, units[no_unit])
+    except IndexError:
+        return "%.0f %s" % (size * ((no_unit - len(units) + 1) * 1000.), units[-1] )
+
+
+class History( BaseUIController ):
+    """
+    Class defining functions used by reports to make requests to get
+    informations and fill templates before being displayed.
+    The name of function must be the same as as the field "action" of
+    the "href" dict, in .mako templates (templates/webapps/reports).
+    """
+    @web.expose
+    def history_and_dataset_per_user( self, trans, **kwd ):
+        """
+        fill history_and_dataset_per_user.mako template with:
+            - user email
+            - the number of history and their size
+            - the number of dataset
+        """
+        message = escape( util.restore_text( kwd.get( 'message', '' ) ) )
+        user_cutoff = int( kwd.get( 'user_cutoff', 60 ) )
+
+        # sort by history space, or by user mail or by number of history/dataset
+        sort_by = kwd.get( 'sorting', 'User' )
+        sorting = 0 if sort_by == 'User' else 1 if sort_by == "HSort" else 2 if sort_by == "DSort" else 3
+        descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1
+
+        # select count (h.id) as history, u.email as email
+        # from history h, galaxy_user u
+        # where h.user_id = u.id and h.deleted='f'
+        # group by email order by email desc
+        histories = sa.select(
+            (sa.func.count( galaxy.model.History.table.c.id ).label( 'history' ),
+             galaxy.model.User.table.c.email.label( 'email' )),
+            from_obj=[sa.outerjoin(galaxy.model.History.table, galaxy.model.User.table)],
+            whereclause=and_(galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id,
+                             galaxy.model.History.table.c.deleted == 'f'),
+            group_by=['email'],
+            order_by=[ sa.desc( 'email' ), 'history' ] )
+
+        # select u.email, count(d.id)
+        # from galaxy_user u, dataset d, history_dataset_association hd,history h
+        # where d.id=hd.dataset_id and h.id=hd.history_id and u.id = h.user_id and h.deleted='f'
+        # group by u.email;
+        datasets = sa.select(
+            (sa.func.count( galaxy.model.Dataset.table.c.id ).label( 'dataset' ),
+             sa.func.sum( galaxy.model.Dataset.table.c.total_size ).label( 'size' ),
+             galaxy.model.User.table.c.email.label( 'email' ) ),
+            from_obj=[ galaxy.model.User.table,
+                       galaxy.model.Dataset.table,
+                       galaxy.model.HistoryDatasetAssociation.table,
+                       galaxy.model.History.table],
+            whereclause=and_(galaxy.model.Dataset.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.dataset_id,
+                             galaxy.model.History.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.history_id,
+                             galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id,
+                             galaxy.model.History.table.c.deleted == 'f'),
+            group_by=[ 'email' ] )
+
+        # execute requests, replace None fields by "Unknown"
+        # transform lists to dict with email as key and
+        # number of (history/dataset)/size of history as value
+        histories = dict( [ ( _.email if _.email is not None else "Unknown", int( _.history ) )
+                            for _ in histories.execute() ] )
+        datasets = dict( [ ( _.email if _.email is not None else "Unknown", (int( _.dataset ), int( _.size )) )
+                           for _ in datasets.execute() ] )
+
+        sorting_functions = [
+            lambda first, second: descending if first[0].lower() > second[0].lower() else -descending,
+            lambda first, second: descending if histories.get(first, 0) < histories.get(second, 0) else -descending,
+            lambda first, second: descending if datasets.get(first, [0])[0] < datasets.get(second, [0])[0] else -descending,
+            lambda first, second: descending if datasets.get(first, [0, 0])[1] < datasets.get(second, [0, 0])[1] else -descending
+        ]
+
+        # fetch all users
+        users = list(set(histories.keys()) | set(datasets.keys()))
+
+        # sort users depending on sort function, defined by user choices
+        users.sort(sorting_functions[sorting])
+        if user_cutoff > 0:
+            users = users[:user_cutoff]
+
+        # to keep ordered
+        data = collections.OrderedDict()
+        for user in users:
+            dataset = datasets.get(user, [0, 0])
+            history = histories.get(user, 0)
+            data[user] = ("%d (%s)" % ( history, int_to_octet( dataset[1] ) ), dataset[0] )
+
+        return trans.fill_template( '/webapps/reports/history_and_dataset_per_user.mako',
+                                    data=data,
+                                    user_cutoff=user_cutoff,
+                                    sorting=sorting,
+                                    descending=descending,
+                                    message=message )
+
+    @web.expose
+    def history_and_dataset_type( self, trans, **kwd ):
+        """
+        fill history_and_dataset_type.mako template with:
+            - the name of history
+            - the number of dataset foreach type
+        """
+        message = escape( util.restore_text( kwd.get( 'message', '' ) ) )
+        user_cutoff = int( kwd.get( 'user_cutoff', 60 ) )
+        descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1
+        user_selection = kwd.get( 'user_selection', None)
+
+        # select d.state, h.name
+        # from dataset d, history h , history_dataset_association hda
+        # where hda.history_id=h.id and hda.dataset_id=d.id order by h.state;
+        from_obj = [ galaxy.model.Dataset.table, galaxy.model.History.table, galaxy.model.HistoryDatasetAssociation.table]
+        if user_selection is not None:
+            from_obj.append( galaxy.model.User.table )
+            whereclause = and_( galaxy.model.Dataset.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.dataset_id,
+                                galaxy.model.History.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.history_id,
+                                galaxy.model.User.table.c.id == galaxy.model.History.table.c.user_id,
+                                galaxy.model.User.table.c.email == user_selection )
+        else:
+            whereclause = and_(galaxy.model.Dataset.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.dataset_id,
+                               galaxy.model.History.table.c.id == galaxy.model.HistoryDatasetAssociation.table.c.history_id)
+        histories = sa.select( ( galaxy.model.Dataset.table.c.state.label( 'state' ),
+                                 galaxy.model.History.table.c.name.label( 'name' ) ),
+                               from_obj=from_obj,
+                               whereclause=whereclause,
+                               order_by=[ 'name' ] )
+
+        # execute requests, replace None fields by "Unknown"
+        data = [ ( _.name if _.name is not None else "NoNamedHistory", _.state )
+                 for _ in histories.execute() ]
+
+        # sort by names descending or ascending
+        data.sort(lambda first, second: descending if first[0].lower() > second[0].lower() else -descending)
+
+        # fetch names in the first list and status in the second
+        if data:
+            names, status = zip( *tuple(data) )
+        else:
+            names, status = [], []
+
+        possible_status = {"ok": 0, "upload": 1, "paused": 2, "queued": 3, "error": 4, "discarded": 5}
+        number_of_possible_status = len( possible_status ) + 1  # + 1 to handle unknown status!
+
+        # to keep ordered
+        datas = collections.OrderedDict()
+        for no, name in enumerate(names):
+            if name not in datas:
+                if user_cutoff > 0:
+                    if len( datas ) == user_cutoff:
+                        break
+                # creation of a list containing the number of each status
+                datas[name] = ['-'] * number_of_possible_status
+            # to not execute it several times, we put it in a variable...
+            no_status = possible_status.get(status[no], 6)
+            if datas[name][no_status] == '-':
+                datas[name][no_status] = 0
+            datas[name][no_status] += 1
+
+        return trans.fill_template( '/webapps/reports/history_and_dataset_type.mako',
+                                    data=datas,
+                                    user_cutoff=user_cutoff,
+                                    descending=descending,
+                                    message=message )
@@ -0,0 +1,365 @@
+
+import collections
+import logging
+import galaxy.model
+import sqlalchemy as sa
+
+from galaxy import util
+from galaxy.web.base.controller import BaseUIController, web
+
+from sqlalchemy import and_
+from datetime import timedelta
+from markupsafe import escape
+
+log = logging.getLogger( __name__ )
+
+
+def int_to_octet(size):
+    try:
+        size = float(size)
+    except ValueError:
+        return "???"
+    except TypeError:
+        if size is None:
+            return "0 o"
+        return "???"
+    units = ("o", "Ko", "Mo", "Go", "To")
+    no_unit = 0
+    while (size >= 1000):
+        size /= 1000.
+        no_unit += 1
+    try:
+        return "%.2f %s" % ( size, units[no_unit] )
+    except IndexError:
+        return "%.0f %s" % ( size * ( ( no_unit - len( units ) + 1 ) * 1000. ), units[-1] )
+
+
+class Tools( BaseUIController ):
+    """
+    Class defining functions used by reports to make requests to get
+    informations and fill templates before being displayed.
+    The name of function must be the same as as the field "action" of
+    the "href" dict, in .mako templates (templates/webapps/reports).
+    """
+
+    def formatted(self, date, colored=False):
+        splited = str(date).split(',')
+        if len(splited) == 2:
+            returned = "%s %dH" % (splited[0], int(splited[1].split(':')[0]))
+            if colored:
+                return '<font color="red">' + returned + '</font>'
+            return returned
+        else:
+            splited = tuple([float(_) for _ in str(date).split(':')])
+            if splited[0]:
+                returned = '%d h. %d min.' % splited[:2]
+                if colored:
+                    return '<font color="orange">' + returned + '</font>'
+                return returned
+            if splited[1]:
+                return "%d min. %d sec." % splited[1:3]
+            return "%.1f sec." % splited[2]
+
+    @web.expose
+    def tools_and_job_state( self, trans, **kwd ):
+        """
+        fill tools_and_job_state.mako template with
+            - the name of the tool
+            - the number of jobs using this tool in state 'ok'
+            - the number of jobs using this tool in error
+        """
+
+        message = escape( util.restore_text( kwd.get( 'message', '' ) ) )
+        user_cutoff = int( kwd.get( 'user_cutoff', 60 ) )
+
+        # sort by history space, or by user mail or by number of history/dataset
+        sort_by = kwd.get( 'sorting', 'Tool' )
+        sorting = 0 if sort_by == 'Tool' else 1 if sort_by == 'ok' else 2
+        descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1
+        sort_functions = ( lambda first, second: descending if first.lower() > second.lower() else -descending,
+                           lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) > tools_and_jobs_ok.get( second ) else descending,
+                           lambda first, second: -descending if tools_and_jobs_error.get( first, 0 ) > tools_and_jobs_error.get( second, 0 ) else descending )
+
+        data = collections.OrderedDict()
+
+        # select count(id), tool_id from job where state='ok' group by tool_id;
+        tools_and_jobs_ok = sa.select( ( galaxy.model.Job.table.c.tool_id .label( 'tool' ),
+                                         sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ),
+                                       from_obj=[ galaxy.model.Job.table],
+                                       whereclause=(galaxy.model.Job.table.c.state == 'ok'),
+                                       group_by=[ 'tool' ] )
+
+        # select count(id), tool_id from job where state='error' group by tool_id;
+        tools_and_jobs_error = sa.select( ( galaxy.model.Job.table.c.tool_id .label( 'tool' ),
+                                            sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ),
+                                          from_obj=[ galaxy.model.Job.table],
+                                          whereclause=(galaxy.model.Job.table.c.state == 'error'),
+                                          group_by=[ 'tool' ] )
+
+        tools_and_jobs_ok = dict( list( tools_and_jobs_ok.execute() ) )
+        tools_and_jobs_error = dict( list( tools_and_jobs_error.execute() ) )
+
+        # select each job name one time
+        tools = list(set(tools_and_jobs_ok.keys()) | set(tools_and_jobs_error.keys()))
+        tools.sort(sort_functions[ sorting ])
+
+        for tool in tools:
+            data[tool] = (str( tools_and_jobs_ok.get(tool, '-') ), str( tools_and_jobs_error.get(tool, '-') ) )
+
+        return trans.fill_template( '/webapps/reports/tools_and_job_state.mako',
+                                    data=data,
+                                    user_cutoff=user_cutoff,
+                                    sorting=sorting,
+                                    descending=descending,
+                                    message=message )
+
+    @web.expose
+    def tools_and_job_state_per_month(self, trans, **kwd ):
+        """
+        fill tools_and_job_state_per_month.mako template with
+            - the name of the tool
+            - the number of jobs using this tool in state 'ok'
+            - the number of jobs using this tool in error
+        """
+
+        message = escape( util.restore_text( kwd.get( 'message', '' ) ) )
+        user_cutoff = int( kwd.get( 'user_cutoff', 60 ) )
+
+        # sort by history space, or by user mail or by number of history/dataset
+        # sort_by = kwd.get( 'sorting', 'Tool' )
+        # sorting = 0 if sort_by == 'Tool' else 1 if sort_by == 'ok' else 2
+        # descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1
+        tool = kwd.get( 'tool', None)
+
+        if tool is None:
+            raise TypeError("Tool can't be None")
+
+        data = collections.OrderedDict()
+
+        # select count(id), create_time from job where state='ok' and tool_id=$tool group by date;
+        date_and_jobs_ok = sa.select( ( sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ),
+                                        sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ),
+                                      from_obj=[ galaxy.model.Job.table],
+                                      whereclause=and_( galaxy.model.Job.table.c.state == 'ok', galaxy.model.Job.table.c.tool_id == tool ),
+                                      group_by=[ 'date' ] )
+
+        # select count(id), create_time from job where state='error' and tool_id=$tool group by date;
+        date_and_jobs_error = sa.select( ( sa.func.date( galaxy.model.Job.table.c.create_time ).label( 'date' ),
+                                           sa.func.count( galaxy.model.Job.table.c.id ).label( 'job' ) ),
+                                         from_obj=[ galaxy.model.Job.table],
+                                         whereclause=and_( galaxy.model.Job.table.c.state == 'error', galaxy.model.Job.table.c.tool_id == tool ),
+                                         group_by=[ 'date' ] )
+
+        # sort_functions = (lambda first, second: descending if first.lower() > second.lower() else -descending,
+        #  lambda first, second: -descending if tools_and_jobs_ok.get( first, 0 ) >
+        #  tools_and_jobs_ok.get( second ) else descending,
+        #   lambda first, second: -descending if tools_and_jobs_error.get( first, 0 ) >
+        #       tools_and_jobs_error.get( second, 0 ) else descending)
+
+        date_and_jobs_ok = dict( list( date_and_jobs_ok.execute() ) )
+        date_and_jobs_error = dict( list( date_and_jobs_error.execute() ) )
+
+        # select each date
+        dates = list(set(date_and_jobs_ok.keys()) | set(date_and_jobs_error.keys()))
+        dates.sort(reverse=True)
+        for date in dates:
+            date_key = date.strftime( "%B %Y" )
+            if date_key not in data:
+                data[date_key] = [int( date_and_jobs_ok.get(date, 0) ), int( date_and_jobs_error.get(date, 0) ) ]
+            else:
+                data[date_key][0] += int( date_and_jobs_ok.get( date, 0 ) )
+                data[date_key][1] += int( date_and_jobs_error.get( date, 0 ) )
+
+        return trans.fill_template( '/webapps/reports/tools_and_job_state_per_month.mako',
+                                    data=data,
+                                    tool=tool,
+                                    user_cutoff=user_cutoff,
+                                    message=message )
+
+    @web.expose
+    def tool_execution_time(self, trans, **kwd):
+        """
+        Fill the template tool_execution_time.mako with informations:
+            - Tool name
+            - Tool average execution time
+            - last job execution time
+            - min and max execution time
+        """
+
+        # liste des tools + temps moyen d'exec du job + temps d'execution du dernier job + tps min et max / mois (?)
+        user_cutoff = int(kwd.get("user_cutoff", 60))
+        sort_by = kwd.get("sort_by", "tool")
+        descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1
+        sort_by = 0 if sort_by == "tool" else 1 if sort_by == "avg" else 2 if sort_by == "min" else 3
+        color = True if kwd.get("color", '') == "True" else False
+
+        data = {}
+        ordered_data = collections.OrderedDict()
+
+        def field_sort(first, second, field):
+            descending if data[first][field] < data[second][field] else -descending
+
+        sort_functions = [
+            lambda first, second: -descending if first.lower() < second.lower() else descending,
+            lambda first, second: field_sort(first, second, "avg"),
+            lambda first, second: field_sort(first, second, "min"),
+            lambda first, second: field_sort(first, second, "max")]
+
+        jobs_times = sa.select( ( galaxy.model.Job.table.c.tool_id.label( "name" ),
+                                  galaxy.model.Job.table.c.create_time.label("create_time"),
+                                  galaxy.model.Job.table.c.update_time.label("update_time"),
+                                  galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time ),
+                                from_obj=[galaxy.model.Job.table])
+
+        jobs_times = [(name, (create, update, time)) for name, create, update, time in jobs_times.execute()]
+        for tool, attr in jobs_times:
+            if tool not in data:
+                data[tool] = { "last": [(attr[1], attr[0])], "avg": [attr[2]] }
+            else:
+                data[tool]["last"].append((attr[1], attr[0]))
+                data[tool]["avg"].append(attr[2])
+
+        for tool in data:
+            data[tool]["min"] = min(data[tool]["avg"])
+            data[tool]["max"] = max(data[tool]["avg"])
+            last = max(data[tool]["last"])
+            data[tool]["last"] = last[0] - last[1]
+            data[tool]["avg"] = sum(data[tool]["avg"], timedelta()) / len(data[tool]["avg"])
+
+        tools = data.keys()
+        if user_cutoff:
+            tools = tools[:user_cutoff]
+        tools.sort(sort_functions[sort_by])
+        for tool in tools:
+            ordered_data[tool] = { "min": self.formatted(data[tool]["min"], color),
+                                   "max": self.formatted(data[tool]["max"], color),
+                                   "avg": self.formatted(data[tool]["avg"], color),
+                                   "last": self.formatted(data[tool]["last"], color) }
+
+        return trans.fill_template( '/webapps/reports/tool_execution_time.mako',
+                                    data=ordered_data,
+                                    descending=descending,
+                                    user_cutoff=user_cutoff,
+                                    sort_by=sort_by )
+
+    @web.expose
+    def tool_execution_time_per_month(self, trans, **kwd):
+        """
+        Fill the template tool_execution_time_per_month.mako with informations:
+            - Tool average execution time
+            - last job execution time
+            - min and max execution time
+        """
+
+        # liste des tools + temps moyen d'exec du job + temps d'execution du dernier job + tps min et max / mois(?)
+        user_cutoff = int(kwd.get("user_cutoff", 60))
+        sort_by = kwd.get("sort_by", "month")
+        descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1
+        sort_by = 0 if sort_by == "month" else 1 if sort_by == "min" else 2 if sort_by == "max" else 3
+        tool = kwd.get("tool", None)
+        color = True if kwd.get("color", '') == "True" else False
+
+        if tool is None:
+            raise ValueError("Tool can't be None")
+
+        ordered_data = collections.OrderedDict()
+        sort_functions = [(lambda first, second, i=i: descending if first[i] < second[i] else -descending) for i in range(4)]
+
+        jobs_times = sa.select( ( sa.func.date_trunc('month', galaxy.model.Job.table.c.create_time ).label('date'),
+                                  sa.func.max(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time),
+                                  sa.func.avg(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time),
+                                  sa.func.min(galaxy.model.Job.table.c.update_time - galaxy.model.Job.table.c.create_time) ),
+                                from_obj=[galaxy.model.Job.table],
+                                whereclause=galaxy.model.Job.table.c.tool_id == tool,
+                                group_by=['date'] )
+
+        months = list(jobs_times.execute())
+        months.sort(sort_functions[sort_by])
+        if user_cutoff:
+            months = months[:user_cutoff]
+
+        for month in months:
+            ordered_data[str(month[0]).split(' ')[0][:-3]] = ( self.formatted(month[1], color),
+                                                               self.formatted(month[2], color),
+                                                               self.formatted(month[3], color) )
+
+        return trans.fill_template( '/webapps/reports/tool_execution_time_per_month.mako',
+                                    data=ordered_data,
+                                    tool=tool,
+                                    descending=descending,
+                                    user_cutoff=user_cutoff,
+                                    sort_by=sort_by )
+
+    @web.expose
+    def tool_error_messages(self, trans, **kwd):
+        tool_name = kwd.get("tool", None)
+        descending = 1 if kwd.get("descending", 'desc') == "desc" else -1
+        sort_by = 0 if kwd.get("sort_by", "time") == "time" else 1
+        cutoff = int(kwd.get("user_cutoff", 60))
+        sort_functions = ( lambda _a, _b: -descending if counter[_a][1] > counter[_b][1] else descending,
+                           lambda _a, _b: -descending if counter[_a][0] > counter[_b][0] else descending )
+
+        if tool_name is None:
+            raise ValueError("Tool can't be none")
+        tool_errors = [ [unicode(a), b] for a, b in
+                        sa.select((galaxy.model.Job.table.c.stderr, galaxy.model.Job.table.c.create_time),
+                        from_obj=[galaxy.model.Job.table],
+                        whereclause=and_( galaxy.model.Job.table.c.tool_id == tool_name,
+                                          galaxy.model.Job.table.c.state == 'error')).execute() ]
+
+        counter = {}
+        for error in tool_errors:
+            try:
+                error[0] = unicode(error[0].decode("utf-8"))
+                # encoding tested:
+                # latin-1 ; iso-8859-1 ; alien ; cenc ; cp037 ; cp437 ; base64 ; utf-8 ; utf-16 ; ascii ; hex
+            except UnicodeEncodeError:
+                for no, lettre in enumerate(error[0]):
+                    try:
+                        str(lettre.decode("utf-8"))
+                    except UnicodeEncodeError:
+                        try:
+                            error[0] = error[0].replace(error[0][no], '?')
+                        except UnicodeEncodeError:
+                            error[0] = "This error contains special character and can't be displayed."
+                            break
+            if error[0] in counter:
+                counter[error[0]][0] += 1
+            else:
+                counter[error[0]] = [1, error[1]]
+
+        data = collections.OrderedDict()
+        keys = counter.keys()
+        if cutoff:
+            keys = keys[:cutoff]
+        keys.sort(sort_functions[sort_by])
+
+        spaces = [' ', '\t', '    ']
+        for key in keys:
+            new_key = '</br>'.join([_ for _ in key.split('\n') if _ and _ not in spaces])
+            if len(new_key) >= 100:
+                to_replace = []
+                words = key.split('\n')
+                for word in words:
+                    if word in to_replace:
+                        continue
+                    if words.count(word) > 1:
+                        print word
+                        to_replace.append(word)
+                for word in to_replace:
+                    sentence = ("</br>" + word) * 2
+                    count = 2
+                    while sentence + "</br>" + word in new_key:
+                        sentence += "</br>" + word
+                        count += 1
+                    print sentence, count
+                    if sentence in new_key:
+                        new_key = new_key.replace(sentence, '</br>' + word + " [this line in %d times]" % (count))
+            data[new_key] = counter[key]
+
+        return trans.fill_template( "/webapps/reports/tool_error_messages.mako",
+                                    data=data,
+                                    descending=descending,
+                                    tool_name=tool_name,
+                                    sort_by=sort_by,
+                                    user_cutoff=cutoff )
@@ -1,18 +1,18 @@
 import calendar
 import logging
 import operator
-from datetime import datetime, date, timedelta
-
-from markupsafe import escape
-import sqlalchemy as sa
-from sqlalchemy import false
-
 import galaxy.model
+import sqlalchemy as sa
+
 from galaxy import util
 from galaxy.web.base.controller import BaseUIController, web
 from galaxy.webapps.reports.controllers.jobs import sorter
 from galaxy.webapps.reports.controllers.query import ReportQueryBuilder

+from datetime import datetime, date, timedelta
+from markupsafe import escape
+from sqlalchemy import false
+
 log = logging.getLogger( __name__ )


@@ -184,3 +184,33 @@ class Users( BaseUIController, ReportQueryBuilder ):
                                    users=users,
                                    user_cutoff=user_cutoff,
                                    message=message )
+
+    @web.expose
+    def history_per_user( self, trans, **kwd ):
+        message = escape( util.restore_text( kwd.get( 'message', '' ) ) )
+        user_cutoff = int( kwd.get( 'user_cutoff', 60 ) )
+        sorting = 0 if kwd.get( 'sorting', 'User' ) == 'User' else 1
+        descending = 1 if kwd.get( 'descending', 'desc' ) == 'desc' else -1
+        sorting_functions = [
+            lambda first, second: descending if first[0].lower() > second[0].lower() else -descending,
+            lambda first, second: descending if first[1] < second[1] else -descending ]
+
+        req = sa.select(
+            ( sa.func.count( galaxy.model.History.table.c.id ).label( 'history' ),
+              galaxy.model.User.table.c.username.label( 'username' ) ),
+            from_obj=[ sa.outerjoin( galaxy.model.History.table, galaxy.model.User.table ) ],
+            whereclause=galaxy.model.History.table.c.user_id == galaxy.model.User.table.c.id,
+            group_by=[ 'username' ],
+            order_by=[ sa.desc( 'username' ), 'history' ] )
+
+        histories = [ (_.username if _.username is not None else "Unknown", _.history) for _ in req.execute() ]
+        histories.sort( sorting_functions[ sorting ] )
+        if user_cutoff != 0:
+            histories = histories[:user_cutoff]
+
+        return trans.fill_template( '/webapps/reports/history_per_user.mako',
+                                    histories=histories,
+                                    user_cutoff=user_cutoff,
+                                    sorting=sorting,
+                                    descending=descending,
+                                    message=message )
@@ -0,0 +1,76 @@
+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+
+%if message:
+    ${render_msg( message, 'done' )}
+%endif
+
+<div class="report">
+<div class="reportBody">
+    <h3 align="center">Histories and Datasets per User</h3>
+    <h4 align="center">Listed in
+    %if descending == 1:
+        descending
+    %else:
+        ascending
+    %endif
+    order by
+    %if sorting == 0:
+        Users
+    %elif sorting == 1:
+        number of History
+    %elif sorting == 2:
+        number of Dataset
+    %else:
+        History Space
+    %endif
+    </h4>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        <tr>
+            <td>
+                <form method="post" controller="users" action="history_and_dataset_per_user">
+                    <p>
+                        Top <input type="textfield" value="${user_cutoff}" size="3" name="user_cutoff"> shown (0 = all).
+                        </br>
+                        Sort:
+                        <select value="${sorting}" size="4" name="sorting">
+                            <option value="User"> by username </option>
+                            <option value="size"> by history space </option>
+                            <option value="HSort"> by number of history </option>
+                            <option value="DSort"> by number of dataset </option>
+                        </select>
+                        <select value="${descending}" size="3" name="descending">
+                            <option value="desc"> descending </option>
+                            <option value="asc"> ascending </option>
+                        </select>
+                        </br>
+                        <button name="action" value="commit">Sort my Data!</button>
+                    </p>
+                </form>
+            </td>
+        </tr>
+    </table>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        %if data:
+            <tr class="header">
+                <td>User</td>
+                <td>Number of History</td>
+                <td>Number of Dataset</td>
+            </tr>
+            <% odd = False%>
+            %for user in data:
+                %if odd:
+                    <tr class="odd_row">
+                %else:
+                    <tr class="tr">
+                %endif
+                <td><a href="history_and_dataset_type?user_selection=${user}">${user}</a></td>
+                <td>${data[user][0]}</td>
+                <td>${data[user][1]}</td>
+                </tr>
+                <% odd = not odd %>
+            %endfor
+        %endif
+    </table>
+</div>
+</div>
@@ -0,0 +1,75 @@
+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+
+%if message:
+    ${render_msg( message, 'done' )}
+%endif
+
+<div class="report">
+<div class="reportBody">
+    <h3 align="center">States of Datasets per History</h3>
+    <h4 align="center">Listed in
+    %if descending == 1:
+        descending
+    %else:
+        ascending
+    %endif
+    order by History name
+    </h4>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        <tr>
+            <td>
+                <form method="post" controller="users" action="history_and_dataset_type">
+                    <p>
+                        Top <input type="textfield" value="${user_cutoff}" size="3" name="user_cutoff"> shown (0 = all).
+                        </br>
+                        Sort:
+                        <select value="${descending}" size="3" name="descending">
+                            <option value="desc"> descending </option>
+                            <option value="asc"> ascending </option>
+                        </select>
+                        </br>
+                        <button name="action" value="commit">Sort my Data!</button>
+                    </p>
+                </form>
+            </td>
+        </tr>
+    </table>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        %if data:
+            <tr class="header">
+                <td>History name</td>
+                <td>Dataset in state 'ok'</td>
+                <td>Dataset in state 'upload'</td>
+                <td>Dataset paused</td>
+                <td>Dataset queued</td>
+                <td>Dataset in error</td>
+                <td>Dataset discarded</td>
+
+                <!-- <td>Dataset in other status</td>  do you want to show other status?-->
+
+            </tr>
+            <% odd = False%>
+            %for name in data:
+                %if odd:
+                    <tr class="odd_row">
+                %else:
+                    <tr class="tr">
+                %endif
+                <td>${name}</td>
+                <td>${data[name][0]}</td>
+                <td>${data[name][1]}</td>
+                <td>${data[name][2]}</td>
+                <td>${data[name][3]}</td>
+                <td>${data[name][4]}</td>
+                <td>${data[name][5]}</td>
+
+                <!-- <td>${data[name][6]}</td>  do you want to show other status?-->
+
+                </tr>
+                <% odd = not odd %>
+            %endfor
+        %endif
+    </table>
+</div>
+</div>
@@ -0,0 +1,68 @@
+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+
+%if message:
+    ${render_msg( message, 'done' )}
+%endif
+
+<div class="report">
+<div class="reportBody">
+    <h3 align="center">Histories per User</h3>
+    <h4 align="center">Listed in
+    %if descending == 1:
+        descending
+    %else:
+        ascending
+    %endif
+    order by
+    %if sorting == 0:
+        Users
+    %else:
+        Number
+    %endif
+    </h4>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        <tr>
+            <td>
+                <form method="post" controller="users" action="history_per_user">
+                    <p>
+                        Top <input type="textfield" value="${user_cutoff}" size="3" name="user_cutoff"> shown (0 = all).
+                        </br>
+                        Sort:
+                        <select value="${sorting}" size="3" name="sorting">
+                            <option value="User"> by username </option>
+                            <option value="Number"> by number </option>
+                        </select>
+                        <select value="${descending}" size="3" name="descending">
+                            <option value="desc"> descending </option>
+                            <option value="asc"> ascending </option>
+                        </select>
+                        </br>
+                        <button name="action" value="commit">Go</button>
+                    </p>
+                </form>
+            </td>
+        </tr>
+    </table>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        %if histories:
+            <tr class="header">
+                <td>User</td>
+                <td>Number of History</td>
+            </tr>
+            <% ctr = 0 %>
+            %for history in histories:
+                %if ctr % 2 == 1:
+                    <tr class="odd_row">
+                %else:
+                    <tr class="tr">
+                %endif
+                    <td>${history[0]}</td>
+                    <td>${history[1]}</td>
+                </tr>
+                <% ctr += 1 %>
+            %endfor
+        %endif
+    </table>
+</div>
+</div>
@@ -9,7 +9,7 @@
 </%def>

 <%def name="stylesheets()">
-    ${parent.stylesheets()}    
+    ${parent.stylesheets()}
    ## Include "base.css" for styling tool menu and forms (details)
    ${h.css( "base", "autocomplete_tagging", "tool_menu" )}

@@ -63,12 +63,23 @@
                <div class="toolSectionPad"></div>
                <div class="toolSectionPad"></div>
                <div class="toolSectionTitle">
-                    <span>Sample Tracking</span>
+                    <span>Histories</span>
                </div>
                <div class="toolSectionBody">
                    <div class="toolSectionBg">
-                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='sample_tracking', action='per_month_all' )}">Sequencing requests per month</a></div>
-                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='sample_tracking', action='per_user' )}">Sequencing requests per user</a></div>
+                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='history', action='history_and_dataset_per_user' )}">Histories and Datasets per User</a></div>
+                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='history', action='history_and_dataset_type' )}">States of Datasets per History</a></div>
+                    </div>
+                </div>
+                <div class="toolSectionPad"></div>
+                <div class="toolSectionPad"></div>
+                <div class="toolSectionTitle">
+                    <span>Tools</span>
+                </div>
+                <div class="toolSectionBody">
+                    <div class="toolSectionBg">
+                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='tools', action='tools_and_job_state' )}">States of Jobs per Tool</a></div>
+                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='tools', action='tool_execution_time' )}">Execution Time per Tool</a></div>
                    </div>
                </div>
                <div class="toolSectionPad"></div>
@@ -93,6 +104,7 @@
                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='users', action='registered_users' )}">Registered users</a></div>
                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='users', action='last_access_date', sort_id='default', order='default' )}">Date of last login</a></div>
                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='users', action='user_disk_usage', sort_id='default', order='default' )}">User disk usage</a></div>
+                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='users', action='history_per_user', sort_id='default', order='default' )}">Number of History per user</a></div>
                    </div>
                </div>
                <div class="toolSectionPad"></div>
@@ -105,8 +117,19 @@
                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='system', action='index' )}">Disk space maintenance</a></div>
                    </div>
                </div>
+                <div class="toolSectionPad"></div>
+                <div class="toolSectionPad"></div>
+                <div class="toolSectionTitle">
+                    <span>Sample Tracking</span>
+                </div>
+                <div class="toolSectionBody">
+                    <div class="toolSectionBg">
+                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='sample_tracking', action='per_month_all' )}">Sequencing requests per month</a></div>
+                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='sample_tracking', action='per_user' )}">Sequencing requests per user</a></div>
+                    </div>
+                </div>
            </div>
-        </div>    
+        </div>
    </div>
 </%def>

@@ -0,0 +1,87 @@
+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+
+%if message:
+    ${render_msg( message, 'done' )}
+%endif
+
+<div class="toolForm">
+    <h3 align="center">${tool_name} in error's stderr message</h3>
+    <h4 align="center">Listed in 
+    %if descending == 1:
+        descending
+    %else:
+        ascending
+    %endif
+    order by number of error of each type
+    </h4>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        <tr>
+            <td>
+                <form method="post" controller="users" action="tool_error_messages?tool=${tool_name}">
+                    <p>
+                        Top <input type="textfield" value="${user_cutoff}" size="3" name="user_cutoff"> shown (0 = all).
+                        </br>
+                        Sort:
+                        <select value="${descending}" size="2" name="descending">
+                            <option value="desc"> descending </option>
+                            <option value="asc"> ascending </option>
+                        </select>
+                        by:
+                        <select value="${sort_by}" size="2" name="sort_by">
+                            <option value="time"> date of the error </option>
+                            <option value="nb"> number of identical error </option>
+                        </select>
+                        </br>
+                        <button name="action" value="commit">Sort my Data!</button>
+                    </p>
+                </form>
+            </td>
+        </tr>
+    </table>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5"  style="border-collapse:collapse;">
+        %if data:
+            <tr class="header">
+                <td>Tool error</td>
+                <td>Number of error like this</td>
+                <td>Date of this error</td>
+            </tr>
+            <% odd = False%>
+            %for error in data:
+                %if odd:
+                    <tr class="odd_row">
+                %else:
+                    <tr class="tr">
+                %endif
+                %if len (error.split ('</br>')) < 7:
+                    <td>${error}</td>
+                %else:
+                    <style>
+                        .content{
+                            height:100px;
+                            width:600px;
+                            overflow:hidden;
+                            text-overflow:ellipsis;}
+                        input[type='checkbox'] { visibility: hidden; position: absolute; }
+                        input[type='checkbox']:checked + .content { height: auto; width: auto;}
+                    </style>
+                    <td>
+                        <label>
+                            <input type="checkbox" />
+                            <div class="content">
+                                <span class="hidden">
+                                    ${error.replace ('</br>'.join (error.split ('</br>')[2:-3]), '...') + \
+                                    '</br>' + '-' * 25 + ' Extended error ' + '-' * 25 + '</br>' + \
+                                    error + '</br>' + '-' * 66}
+                                </span>
+                            </div>
+                        </label>
+                    </td>
+                %endif
+                <td>${data[error][0]}</td>
+                <td>${str(data[error][1])[:11]}</td>
+                <% odd = not odd %>
+            %endfor
+        %endif
+    </table>
+</div>
@@ -0,0 +1,79 @@
+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+
+
+<div class="report">
+<div class="reportBody">
+    <h3 align="center">Execution Time per Tool</h3>
+    <h4 align="center">Listed in
+    %if descending == 1:
+        descending
+    %else:
+        ascending
+    %endif
+    order by
+    %if sort_by == 0:
+        Tool
+    %elif sort_by == 1:
+        average time
+    %elif sort_by == 2:
+        min time
+    %elif sort_by == 3:
+        max time
+    %endif
+    </h4>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        <tr>
+            <td>
+                <form method="post" controller="users" action="tool_execution_time">
+                    <p>
+                        Top <input type="textfield" value="${user_cutoff}" size="3" name="user_cutoff"> shown (0 = all).
+                        </br>
+                        Sort:
+                        <select value="${sort_by}" size="4" name="sort_by">
+                            <option value="tool"> by Tool </option>
+                            <option value="avg"> average time </option>
+                            <option value="min"> min time </option>
+                            <option value="max"> max time </option>
+                        </select>
+                        <select value="${descending}" size="3" name="descending">
+                            <option value="desc"> descending </option>
+                            <option value="asc"> ascending </option>
+                        </select>
+                        <input type="checkbox" name="color" value="True">
+                            Highlight big times of execution</br>
+                        <button name="action" value="commit">Sort my Data!</button>
+                    </p>
+                </form>
+            </td>
+        </tr>
+    </table>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        %if data:
+            <tr class="header">
+                <td>Tool</td>
+                <td>Average time of execution</td>
+                <td>Last job's time of execution</td>
+                <td>Min time of execution</td>
+                <td>Max time of execution</td>
+            </tr>
+            <% odd = False%>
+            %for tool in data:
+                %if odd:
+                    <tr class="odd_row">
+                %else:
+                    <tr class="tr">
+                %endif
+                <td>
+                    <a href=tool_execution_time_per_month?tool=${tool}>${tool}</a>
+                </td>
+                <td>${data[tool]["avg"]}</td>
+                <td>${data[tool]["last"]}</td>
+                <td>${data[tool]["min"]}</td>
+                <td>${data[tool]["max"]}</td>
+                <% odd = not odd %>
+            %endfor
+        %endif
+    </table>
+</div>
+</div>
@@ -0,0 +1,77 @@
+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+
+
+<div class="report">
+<div class="reportBody">
+    <h3 align="center">Execution Time for ${tool} per Month</h3>
+    <h4 align="center">Listed in
+    %if descending == 1:
+        descending
+    %else:
+        ascending
+    %endif
+    order by
+    %if sort_by == 0:
+        Month
+    %elif sort_by == 1:
+        min time
+    %elif sort_by == 2:
+        max time
+    %elif sort_by == 3:
+        average time
+    %endif
+    </h4>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        <tr>
+            <td>
+                <form method="post" controller="users" action="tool_execution_time_per_month?tool=${tool}">
+                    <p>
+                        Top <input type="textfield" value="${user_cutoff}" size="3" name="user_cutoff"> shown (0 = all).
+                        </br>
+                        Sort:
+                        <select value="${sort_by}" size="4" name="sort_by">
+                            <option value="month"> by Month </option>
+                            <option value="min"> min time </option>
+                            <option value="max"> max time </option>
+                            <option value="avg"> average time </option>
+                        </select>
+                        <select value="${descending}" size="3" name="descending">
+                            <option value="desc"> descending </option>
+                            <option value="asc"> ascending </option>
+                        </select>
+                        <input type="checkbox" name="color" value="True">
+                            Highlight big times of execution</br>
+                        <button name="action" value="commit">Sort my Data!</button>
+                    </p>
+                </form>
+            </td>
+        </tr>
+    </table>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        %if data:
+            <tr class="header">
+                <td>Tool</td>
+                <td>Min time of execution</td>
+                <td>Max time of execution</td>
+                <td>Average time of execution</td>
+            </tr>
+            <% odd = False%>
+            %for month in data:
+                %if odd:
+                    <tr class="odd_row">
+                %else:
+                    <tr class="tr">
+                %endif
+                <td>
+                ${month}
+                </td>
+                <td>${data[month][2]}</td>
+                <td>${data[month][0]}
+                <td>${data[month][1]}</td></td>
+                <% odd = not odd %>
+            %endfor
+        %endif
+    </table>
+</div>
+</div>
@@ -0,0 +1,84 @@
+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+
+%if message:
+    ${render_msg( message, 'done' )}
+%endif
+
+<div class="report">
+<div class="reportBody">
+    <h3 align="center">States of Jobs per Tool</h3>
+    <h4 align="center">Listed in
+    %if descending == 1:
+        descending
+    %else:
+        ascending
+    %endif
+    order by
+    %if sorting == 0:
+        Tool
+    %elif sorting == 1:
+        state Ok
+    %elif sorting == 2:
+        state error
+    %endif
+    </h4>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        <tr>
+            <td>
+                <form method="post" controller="users" action="tools_and_job_state">
+                    <p>
+                        Top <input type="textfield" value="${user_cutoff}" size="3" name="user_cutoff"> shown (0 = all).
+                        </br>
+                        Sort:
+                        <select value="${sorting}" size="3" name="sorting">
+                            <option value="tool"> by Tool </option>
+                            <option value="ok"> state ok </option>
+                            <option value="error"> state error </option>
+                        </select>
+                        <select value="${descending}" size="3" name="descending">
+                            <option value="desc"> descending </option>
+                            <option value="asc"> ascending </option>
+                        </select>
+                        </br>
+                        <button name="action" value="commit">Sort my Data!</button>
+                    </p>
+                </form>
+            </td>
+        </tr>
+    </table>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        %if data:
+            <tr class="header">
+                <td>Tool</td>
+                <td>Jobs ok</td>
+                <td>Job in error</td>
+            </tr>
+            <% odd = False%>
+            %for tool in data:
+                %if odd:
+                    <tr class="odd_row">
+                %else:
+                    <tr class="tr">
+                %endif
+                <td>
+                %if data[tool][0] + data[tool][1] != "--":
+                    <a href=tools_and_job_state_per_month?tool=${tool}>${tool}</a>
+                %else:
+                    ${tool}
+                %endif
+                </td>
+                <td>${data[tool][0]}</td>
+                <td>
+                %if data[tool][1] != '-':
+                    <a href=tool_error_messages?tool=${tool}>${data[tool][1]}</a>
+                %else:
+                    -
+                %endif
+                </td>
+                <% odd = not odd %>
+            %endfor
+        %endif
+    </table>
+</div>
+</div>
@@ -0,0 +1,46 @@
+<%inherit file="/base.mako"/>
+<%namespace file="/message.mako" import="render_msg" />
+
+%if message:
+    ${render_msg( message, 'done' )}
+%endif
+
+<div class="report">
+<div class="reportBody">
+    <h3 align="center">States of Jobs for ${tool}</h3>
+    <h4 align="center">Listed in descending by month</h4>
+    <table align="center" width="70%" class="colored" cellpadding="5" cellspacing="5">
+        %if data:
+            <tr class="header">
+                <td>Month</td>
+                <td>Ok</td>
+                <td>Error</td>
+            </tr>
+            <% odd = False%>
+            %for month in data:
+                %if odd:
+                    <tr class="odd_row">
+                %else:
+                    <tr class="tr">
+                %endif
+                <td>${month}</td>
+                <td>
+                %if data[month][0] == 0:
+                    -
+                %else:
+                    ${data[month][0]}
+                %endif
+                </td>
+                <td>
+                %if data[month][1] == 0:
+                    -
+                %else:
+                    ${data[month][1]}
+                %endif
+                </td>
+                <% odd = not odd %>
+            %endfor
+        %endif
+    </table>
+</div>
+</div>