OwlCyberSecurity - MANAGER

Edit File: lvechartmain.py

# coding=utf-8
#
# Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved
#
# Licensed under CLOUD LINUX LICENSE AGREEMENT
# http://cloudlinux.com/docs/LICENSE.TXT
from __future__ import absolute_import
from __future__ import division
from builtins import range
from clcommon.cpapi.pluginlib import getuser
from lvestats.lib.uidconverter import uid_to_username
from lvestats.lib.lveinfolib import HistoryShow, FIELD_LIMIT
from lvestats.lib.chart import ChartMain
from lvestats.lib.commons import dateutil, sizeutil
from lvestats.lib.chart.util import X_LEGEND_POINTS
from lvestats.lib import lveinfolib_gov
from lvestats.lib.commons import func

def _4k_page_to_bytes(v):
    return (v or 0) * 4 * 1024

SHOW_COLUMNS = ('From', 'aCPU', 'lVMem', 'aPMem', 'lPMem', 'NprocF', 'PMemF', 'VMemF',
                'aIO', 'lIO', 'lIOPS', 'IOPSf',
                'IOf', 'lCPU', 'CPUf', 'aVMem', 'aEP', 'lEP', 'aNproc', 'lNproc',
                'EPf', 'aIOPS', 'mIOPS',)

class LveChart(ChartMain):
    def __init__(self, config):
        super(LveChart, self).__init__('lvechart',
                                       'Creates a chart representing usage pattern for LVE/user',
                                       config)
        self.MINIMAL_TIME_STEP = int(config.get('aggregation_period', 60))
        self.BYTES_IN_MB = 1024 * 1024
        self.governor_mode = func.get_governor_mode()
        self.user_ignore = False

def customize_parser(self, parser):
        is_admin = getuser() == 'root'
        id_user_group = parser.add_mutually_exclusive_group(required=is_admin)

id_user_group.add_argument('--id',
                                   help='LVE id -- will display record only for that LVE id',
                                   dest='user_id',
                                   type=int)
        id_user_group.add_argument('--user',
                                   help='Use username instead of LVE id, and show only record for that user',
                                   dest='user_name',
                                   default=None)
        return parser

def append_dbgov_data_to_dict(self, dictionary, governor_mode_is_all, num, dbcpu_sum, dbio_sum, _lve_time_index):
        """
        :param dictionary: dict to wich we append dbgov data
        :param governor_mode_is_all:
        :param num: number of dbgov data points near this LVEStatsHistory time
        :param dbcpu_sum: sum value of dbgov cpu near this LVEStatsHistory time
        :param dbio_sum: sum value of dbgov io near this LVEStatsHistory time
        :param _lve_time_index: index of lve point
        :return:
        """
        if num > 0:
            if governor_mode_is_all:
                # append mean value
                dictionary["dbcpu"].append(min(dbcpu_sum / num, dictionary['acpu'][_lve_time_index]))
                try:
                    dictionary["dbio"].append(
                        min(dbio_sum / num * self.BYTES_IN_MB, dictionary['aio'][_lve_time_index]))
                except (KeyError, IndexError):
                    # if lve_version <= 4 there is no by_key['aio']
                    pass
            else:
                dictionary["dbcpu"].append(dbcpu_sum / num)
                dictionary["dbio"].append(dbio_sum / num * self.BYTES_IN_MB)
        # when there was no dbgov data near this LVEStatsHistory time
        else:
            dictionary["dbcpu"].append(0)
            dictionary["dbio"].append(0)

@staticmethod
    def make_lvedata_with_idle(lvedata, show_columns, period_from, period_to, time_step):

def get_time_from(index):
            try:
                return int(lvedata[index][show_columns.index('From')])
            except IndexError:
                return 0

lvedata_with_idle = list()  # prepare data for correct show with server idle
        indx_ = 0
        for period_from_ in range(period_from, period_to, time_step):
            if lvedata and period_from_ - time_step / 2 < get_time_from(indx_) <= period_from_ + time_step / 2:
                lvedata_with_idle.append(lvedata[indx_])
                indx_ += 1
            else:
                line_ = list()
                for item in show_columns:
                    if item == 'From':
                        line_.append(period_from_)
                    elif item in FIELD_LIMIT:
                        # calculate limit for limit fields
                        if lvedata:
                            # use previous limit
                            line_.append(lvedata[max(0, indx_ - 1)][show_columns.index(item)])
                        else:
                            line_.append(0)
                    else:
                        line_.append(0)
                lvedata_with_idle.append(line_)
        return lvedata_with_idle

def get_chart_data(self,
                       engine,
                       from_ts,
                       to_ts,
                       server,
                       user_id,
                       show_all=False):
        username = uid_to_username(
            uid=user_id,
            local_server_id=self.cfg.get("server_id", "localhost"),
            server_id=server,
            db_engine=engine
        )
        self.user_ignore = func.get_governor_ignore_for_user(username)

utc_from = dateutil.local_to_gm(from_ts)
        utc_to = dateutil.local_to_gm(to_ts)

period_sec, time_step = self.get_time_step(utc_from, utc_to)

lvedata_with_idle = self.load_data(engine, server, time_step, user_id, utc_from, utc_to)

by_key_ = {col.lower(): [row[col_index] for row in lvedata_with_idle] for col_index, col in
                   enumerate(SHOW_COLUMNS)}
        by_key = LveChart.convert_lvedata_to_dict(by_key_)
        lve_times = by_key['from']
        del by_key['from']
        show_columns = ('ts', 'cpu', 'lcpu', 'read', 'lread', 'write', 'lwrite')
        if self.governor_mode != "none" and not self.user_ignore:
            dbdata = lveinfolib_gov.HistoryShowDBGov(
                engine,
                utc_from,
                utc_to,
                uid=user_id,
                server_id=server,
                show_columns=show_columns,
                cfg=self.cfg,
            ).history_dbgov_show()

data_collected = LveChart.convert_dbdata_to_dict(dbdata, show_columns)
            dbtimes = data_collected['ts']
            del data_collected['ts']

time_step2 = time_step / 2.0
            # append dbgov stats to the nearest HistoryShow points
            for lve_time_index, lve_time in enumerate(lve_times):
                dbcpu, dbio = 0, 0
                tmp_dbtimes_list = dbtimes[:]  # temporary list for iteration
                for dbtime_index, dbtime in enumerate(tmp_dbtimes_list):
                    # check if dbgov data is near this LVEStatsHistory time
                    if dbtime < lve_time + time_step2:
                        dbcpu += data_collected['cpu'].pop(0)
                        dbio += data_collected["read"].pop(0) + data_collected["write"].pop(0)
                        dbtimes.pop(0)
                    else:
                        self.append_dbgov_data_to_dict(
                            by_key,
                            self.governor_mode == "all",
                            dbtime_index,
                            dbcpu,
                            dbio,
                            lve_time_index)
                        break
                else:
                    self.append_dbgov_data_to_dict(
                        by_key,
                        self.governor_mode == "all",
                        len(tmp_dbtimes_list),
                        dbcpu,
                        dbio,
                        lve_time_index)
        return by_key, lve_times, period_sec

def load_data(self, engine, server, time_step, user_id, utc_from, utc_to):
        history_show = HistoryShow(dbengine=engine, period_from=utc_from, period_to=utc_to, uid=user_id,
                                   show_columns=SHOW_COLUMNS, server_id=server, time_unit=time_step)
        lvedata = list(history_show.proceed())
        lvedata_with_idle = self.make_lvedata_with_idle(lvedata, SHOW_COLUMNS, history_show.period_from,
                                                        history_show.period_to, time_step)
        return lvedata_with_idle

def get_time_step(self, utc_from, utc_to):
        dt = utc_to - utc_from
        period_sec = dt.total_seconds()
        return period_sec, self.MINIMAL_TIME_STEP

@staticmethod
    def _add_faults_graph(renderer, data_collected, times, lve_version):
        legend = {
            'CPUf': ('CPUf', '#8DB600'),
            'EPf': ('EPf', 'green',),
            'VMemF': ('VMemF', '#00DDFF'),
            'NprocF': ('NprocF', 'red'),
            'PMemF': ('PMemF', 'blue'),
            'IOf': ('IOf', '#9966CC')}
        if lve_version >= 8:
            legend.update({
                'IOPSf': ('IOPSf', '#007FFF'),
            })
        legend_keys = tuple(key.lower() for key in list(legend.keys()))
        faults_present = any(tuple(any(data_collected[key]) for key in legend_keys))
        message = 'No Faults' if not faults_present else None
        renderer.add_graph(
            data_collected, 'Faults',
            legend=legend, x_values=times, min_y=0,
            y_legend_converter=sizeutil.convert_powers_of_1000_for_graph,
            message=message)

def add_graphs(self, renderer, data_collected, times, lve_version, show_all, is_user=False):
        title_suffix = self._get_title_suffix(data_collected)

is_lcpu_limited = any(data_collected['lcpu'])
        if is_lcpu_limited or show_all:
            if is_user and self.is_normalized_user_cpu and is_lcpu_limited:
                self._normalize_cpu(data_collected)

cpu_legend = {'aCPU': ('average', 'green'), 'lCPU': ('limit', 'red')}
            if show_all and not is_lcpu_limited:
                del cpu_legend['lCPU']
            if len(data_collected['dbcpu']) > 0:
                cpu_legend['dbcpu'] = ('database', 'blue')
            renderer.add_graph(data_collected, 'CPU Usage' + title_suffix,
                               legend=cpu_legend,
                               x_values=times, min_y=0, unit='%',
                               y_legend_converter=int,
                               faults=('CPUf', 'aCPU', 'lCPU'))

is_lvmem_limited = any(data_collected['lvmem'])
        if is_lvmem_limited or show_all:
            vmem_legend = {
                'lVmem': ('limit', 'red', _4k_page_to_bytes),
                'aVmem': ('average', 'green', _4k_page_to_bytes)}
            if show_all and not is_lvmem_limited:
                del vmem_legend['lVmem']
            renderer.add_graph(data_collected, 'Virtual Memory Usage',
                               legend=vmem_legend, min_y=0, x_values=times,
                               y_legend_converter=sizeutil.convert_bytes_for_graph)
        is_lpmem_limited = any(data_collected['lpmem'])
        if is_lpmem_limited or show_all:
            pmem_legend = {
                'lPMem': ('limit', 'red', _4k_page_to_bytes),
                'aPMem': ('average', 'green', _4k_page_to_bytes)}
            if show_all and not is_lpmem_limited:
                del pmem_legend['lPMem']
            renderer.add_graph(data_collected, 'Physical Memory Usage',
                               legend=pmem_legend, x_values=times, min_y=0,
                               y_legend_converter=sizeutil.convert_bytes_for_graph)
        is_lio_limited = any(data_collected['lio'])
        if is_lio_limited or show_all:
            io_legend = {
                'lIO': ('limit', 'red'),
                'aIO': ('average', 'green')}
            if show_all and not is_lio_limited:
                del io_legend['lIO']
            if len(data_collected['dbio']) > 0:
                io_legend['dbio'] = ('database', 'blue')
            renderer.add_graph(data_collected, 'Input/Output Usage' + title_suffix,
                               legend=io_legend, x_values=times, min_y=0,
                               y_legend_converter=sizeutil.convert_bytes_for_graph,
                               unit='/s',
                               faults=('IOf', 'aIO', 'lIO'))
        if lve_version > 6:
            is_liops_limited = any(data_collected['liops'])
            if is_liops_limited or show_all:
                iops_legend = {
                    'lIOPS': ('limit', 'red'),
                    'aIOPS': ('average', 'green')}
                if show_all and not is_liops_limited:
                    del iops_legend['lIOPS']
                renderer.add_graph(data_collected, 'Io operations',
                                   legend=iops_legend, x_values=times, min_y=0,
                                   y_legend_converter=sizeutil.convert_powers_of_1000_for_graph,
                                   unit='/s',
                                   faults=('IOPSf', 'aIOPS', 'lIOPS'))
        is_lep_limited = any(data_collected['lep'])
        if is_lep_limited or show_all:
            ep_legend = {
                'lEp': ('limit', 'red'),
                'aEp': ('average', 'green')}
            if show_all and not is_lep_limited:
                del ep_legend['lEp']
            renderer.add_graph(data_collected, 'Entry Processes',
                               legend=ep_legend, x_values=times, min_y=0,
                               y_legend_converter=sizeutil.convert_powers_of_1000_for_graph)
        is_lnproc_limited = any(data_collected['lnproc'])
        if is_lnproc_limited or show_all:
            nproc_legend = {
                'lNproc': ('limit', 'red'),
                'aNproc': ('average', 'green')}
            if show_all and not is_lnproc_limited:
                del nproc_legend['lNproc']
            renderer.add_graph(data_collected, 'Processes',
                               legend=nproc_legend, x_values=times, min_y=0,
                               y_legend_converter=sizeutil.convert_powers_of_1000_for_graph)

self._add_faults_graph(renderer, data_collected, times, lve_version)
        renderer.add_common_x_legend(times, X_LEGEND_POINTS)

@staticmethod
    def _normalize_cpu(data_collected):
        # Magnify aCPU, dbCPU, lCPU to 100%
        # 1. Find max limit
        max_limit = max(data_collected['lcpu'])
        # 2. Calculate New user's data and limits
        a_cpu_new = []
        l_cpu_new = []
        db_cpu_new = []
        for idx in range(0, len(data_collected['lcpu'])):
            # New limit
            l_cpu_new.append(data_collected['lcpu'][idx] * 100.0 / max_limit)
            # New lve cpu average
            a_cpu_new.append(data_collected['acpu'][idx] * 100.0 / max_limit)
            if len(data_collected['dbcpu']) > 0:
                # New database cpu average
                db_cpu_new.append(data_collected['dbcpu'][idx] * 100.0 / max_limit)
        # Store new data
        data_collected['lcpu'] = l_cpu_new
        data_collected['acpu'] = a_cpu_new
        if db_cpu_new:
            data_collected['dbcpu'] = db_cpu_new

def _get_title_suffix(self, data_collected):
        # <lve use="on|single|off|abusers|all"/>
        if len(data_collected['dbcpu']) == 0 or self.governor_mode == "none" or self.user_ignore:
            title_suffix = ""
        elif self.governor_mode == "all":
            title_suffix = ". DB usage included"
        elif self.governor_mode == "abusers":
            title_suffix = ". DB usage included, only if restricted"
        elif self.governor_mode == "off":
            title_suffix = ". DB usage is not limited, nor accounted for"
        else:
            # single or on
            title_suffix = ". DB usage is not accounted as part of LVE"
        return title_suffix