import logging
from abc import ABCMeta, abstractmethod
from asyncio.log import logger
from datetime import timedelta
from typing import Final, final
from django.apps import AppConfig
from django.conf import settings
from django.template import loader
from django.utils import timezone
from .status import STATUS
# TODO: Make each instance one Analysis Config instance and not some list of dictionaries
[docs]
class AnalysisConfig(AppConfig, metaclass=ABCMeta):
"""
Abstract class handling the common methods of an Analysis and representing all of the analysis
configurations.
"""
logger: Final[logging.Logger] = logging.getLogger("Happyface4")
"""
:meta private:
The logger which can be used to log to. E.g. with ``self.logger.error("..msg..")``.
"""
instance_parallelization: bool = False
"""
Enable or disable parallel execution of instances within this analysis.
When set to True, instances will be processed in parallel using ThreadPoolExecutor,
which can significantly reduce getData execution time for analyses with multiple instances.
**Important**: Only set this to True if you have verified that:
1. Instances do not share database tables (or use proper locking)
2. Instances do not have race conditions in save_data_to_db()
3. Each instance operates on independent data
Default is False for safety.
Example:
.. code-block:: python
class QueuedDataTransfersConfig(AnalysisConfig):
name = "queued_data_transfers"
instance_parallelization = True # Safe - verified no conflicts
instances = [...]
"""
db_data_required: bool = True
"""
Whether ``builddiv`` reads analysis data from the database for the initial
render. Analyses that fetch all their data asynchronously from the client
set this to False to skip the database lookback.
"""
@property
@abstractmethod
def name(self) -> str:
"""
standard name of the analysis is the module name, which is the name of the folder
in which the analysis is located
"""
pass
@property
@abstractmethod
def instances(self) -> list:
"""
List of :class:`dicts <dict>` which contain the info and settings of the
different instances of one Analysis.
Should have at least the following contents:
.. code-block:: python
instances = [
{
# Name of the analysis instance
"name": "happyface4_analysis_instance_1",
# This will get the visible header of the analysis
"verbose_name": "Happyface 4 Analysis Example 1",
# Category of the instance. Each category is displayed in a separate
# site.
"category": "Batch System",
# Url to data source
"source": "example.com",
# A short description of the instance
"description": "Status of the Compute Element Tests",
# If the instance doesn't have an internal status to set and has only
# the status `info`, this flag can be set to True.
# In this case the `get_instance_status()` method doesn't need to be
# specified and `save_data_to_db()` should return `None,None`.
"is_info": True,
# A threshold for the 'critical' and the 'warning' alert level. Putting
# it here is useful if there are different threshold for different
# instances in an analysis.
"critical_thresholds": 0.8,
"warning_thresholds": 0.5,
# The file for the HTML template.
"template": "happyface4_analysis/happyface4_analysis.html",
# The time interval in which the data is fetched. This is an int giving multiples of settings.PULL_INTERVAL
"get_data_every": 1,
}
]
"""
pass
def __init__(self, app_name, app_module):
# calls the __init__ function of the parent class AppConfig
super().__init__(app_name, app_module)
# replaces all whitespaces with underscores and makes all lowercase for the names
# of the analysis and the instances
self.name = self.name.replace(" ", "_").lower()
# Set default for instance parallelization (can be overridden by subclass)
if not hasattr(self, "instance_parallelization"):
self.instance_parallelization = False
# check if instances dicts are complete and if not insert some standard values
for i, inst in enumerate(self.instances):
if not inst.get("name"):
inst["name"] = self.name + "_" + str(i)
else:
# replace whitespaces and make all lowercase
inst["name"] = inst["name"].replace(" ", "_").lower()
if not inst.get("verbose_name"):
inst["verbose_name"] = inst["name"].replace("_", " ").capitalize()
if not inst.get("category"):
inst["category"] = "Uncategorized"
if not inst.get("dt"):
inst["dt"] = settings.PULL_INTERVAL
if not inst.get("source"):
self.logger.error(
"The source variable is missing for analysis {} instance {}.".format(
self.name, inst["name"]
)
)
raise NotImplementedError(
"The source variable is missing for analysis {} instance {}.".format(
self.name, inst["name"]
)
)
if not inst.get("description"):
inst["description"] = "There is no description available yet."
self.logger.warning(
"The description for {}. {} ist missing.".format(
self.name, inst["name"]
)
)
# check if template is set, if not use the class variable to deduce the
# template name
if not inst.get("template"):
inst["template"] = "{0}/{0}.html".format(self.name)
# check if the get_data_every variable is set, if not set it to 1
if not inst.get("get_data_every"):
inst["get_data_every"] = 1
[docs]
@final
def getData(self, instance):
"""
Central function which is called by the global getData command for each instance.
Handles the process of fetching and saving data and calculating the instance status.
:meta private:
:param instance: The dict with the properties of an analysis instance, this is
passed by the global
:class:`getData <~commands.management.commands.getData.Command>` function.
:type instance: dict
"""
from .models import InstanceStatus
data = self.extract_data_from_url(instance)
# checks whether data was fetched and if not sets the status for the actual time
if data:
status, time = self.save_data_to_db(data, instance)
else:
status = STATUS.TECHNICAL_ISSUE
time = timezone.now()
self.logger.error(f"Instance {instance['name']}: Couldn't fetch data.")
# Don't save the status if the save_data_to_db method doesn't return a status
if status is not None:
# saves the alert level
InstanceStatus.objects.update_or_create(
time=time,
instance=instance["name"],
category=instance["category"],
status=status,
)
[docs]
@abstractmethod
def save_data_to_db(self, data, instance):
"""This function takes the fetched data and prepares it for the database and
finally saves it in the database.
This function needs to be overwritten!
The preparation includes removing all data not needed and making all analytical
calculation needed. The more calculation is done here the less calculation needs
to be done when an HTML request comes and therefore the site will load faster.
But remember this function is called every time the global getData function is
called (e.g. every 15min) so the calculations should not be too big.
To save the data to the database (you can define the database tables in the
``models.py``) you need to do something like this:
.. code-block:: python
from . import models
models.ExampleModel.objects.create(instance=instance['name'], \
example_data=prepared_data, ...)
For more information please look into the
`django documentation <https://docs.djangoproject.com/en/6.0/ref/models/>`_.
:param data: This is exactly the data which is returned from the
:func:`extract_data_from_url` function.
:type data: dict or list
:param instance: The dict with the properties of an analysis instance, this is
passed automatically by the :func:`getData` function.
:type instance: dict
:returns:
- **instance_status** - The alert level of the whole instance. This is
important for the display of the alert level in the navigation bar on the
website. ``-2: 'Technical issue', -1: 'info', 0: 'ok', 1: 'warning',
2: 'critical'``
- **time** - The time the data is fetched. This can be the actual time, in
this case :func:`~django.utils.timezone.now` would be the right choice, or
it can be a time which is recorded in the data itself.
:rtype: (int, ~datetime.datetime)
"""
# refresh imports
from . import models
self.logger.error(
"The function save_data_to_db of instance {} doesn't exist.".format(
instance["name"]
)
)
instance_status = None
time = None # timezone.now()
# prepare the fetched data for the dataset and make all analytical calculations
# needed
# only save the data in the db which is really needed
return instance_status, time
[docs]
@final
def builddiv(self, time_of_readout, instance, request):
"""Central function which is called if there is an HTTP request to a category with
this instance in it. Calls the retrieve_data_from_db function and renders the
template with the data obtained from the database. Analyses with
``db_data_required = False`` fetch all data from the client and are rendered
without the database lookback.
:meta private:
:param time_of_readout: The time the data is requested for.
:type time_of_readout: ~datetime.datetime
:param instance: The dict with the properties of an analysis instance.
:type instance: dict
:param request: A django request instance with all the information needed to answer the request.
:type request: ~django.http.HttpRequest
:returns: The rendered HTML code for this analysis.
:rtype: str
"""
# loads the global setting of the timedelta after which new data is fetched. So we
# only need to search for data in the frame time_of_readout-dt to time_of_readout.
dt = instance["dt"]
history_steps = 0
data = None
timestamp = time_of_readout
status = STATUS.TECHNICAL_ISSUE
if not self.db_data_required:
# all data is fetched asynchronously by the client; render without db lookback
if instance.get("is_info", False):
status = STATUS.INFO
else:
status = (
self.get_instance_status(instance=instance, time=time_of_readout)
.latest("time")
.status
)
data = True
else:
# tries to get the data. If the data returned is None, then this tries the to get
# the data dt (mins) in the past and iterates this until some data is found or it
# needs more then 8 iterations.
# An additional warning gets displayed in this case.
while history_steps < 8:
# gets the data for the requested time. The timestep the data was produced can
# differ from the time in the request. So an additional timestamp is passed.
data, timestamp = self.retrieve_data_from_db(
instance, time_of_readout, dt
)
if data and history_steps == 0:
# gets the status of the data to the requested time from the database.
# It could also be calculated from the data, but this would need some time
# and a short response time of the website is more important.
if instance.get("is_info", False):
status = STATUS.INFO
else:
status = (
self.get_instance_status(
instance=instance, time=time_of_readout
)
.latest("time")
.status
)
break
elif data and history_steps > 0:
status = STATUS.WARNING
break
else:
time_of_readout -= dt
history_steps += 1
# the context dict which is passed to the template
context = {
"status": status if data else STATUS.TECHNICAL_ISSUE,
"old_data_warning": (
{
"hours": (history_steps * dt).seconds // 3600,
"min": ((history_steps * dt).seconds // 60) % 60,
}
if history_steps and data
else None
),
"data": data,
"instance": instance,
"timestamp": timestamp,
"analysis_name": self.name,
"data_sources": (
instance["source"]
if type(instance["source"]) == list
else [instance["source"]]
),
}
# Load and render the template with the context dict.
template = loader.get_template(instance["template"])
return template.render(context, request)
[docs]
@abstractmethod
def retrieve_data_from_db(self, instance, time_of_readout, dt):
"""This function is responsible for loading the data of this instance from the
database for the desired time.
The filtering of the database for the desired time can be defined in the
``managers.py`` file, so this function only needs to check if data is available at
that time.
Then the data must be returned together with the timestamp. The timestamp shows
when the data is from.
The default implementations of this abstract method only log an error and return
``None``; it needs to be overwritten unless the analysis sets
``db_data_required = False`` and fetches all data from the client.
:param instance: The dict with the properties of an analysis instance the data was
requested for.
:type instance: dict
:param time_of_readout: The date and time the data was requested for.
:type time_of_readout: ~datetime.datetime
:param dt: Specifies the time range in which to search for data, i.e. data between
``time_of_readout - dt`` and ``time_of_readout`` is considered.
:type dt: ~datetime.timedelta
:returns:
- **data** - A dict or
:class:`Django Queryset <django.db.models.query.QuerySet>` with the data.
This is directly loaded in the template and has to be prepared for it.
- **timestamp** - The date and time the data is from.
:rtype: (dict or ~django.db.models.query.QuerySet, ~datetime.datetime)
"""
from . import models
self.logger.error(
"The function retrieve_data_from_db of instance {} doesn't exist.".format(
instance["name"]
)
)
data = None
timestamp = None
return data, timestamp
[docs]
def get_instance_status(self, instance, time, time_range=None):
"""This function gets or calculates the status of an instance.
The base implementation queries the stored instance statuses; subclasses
may override it to compute the status from the data instead.
:param instance: The dict with the properties of an analysis instance the instance
status was requested for.
:type instance: dict
:param time: The time in which the data was retrieved for which the
instance status is requested.
:type time: ~datetime.datetime
:param time_range: Specifies the time range in which to search for statuses, i.e.
statuses between ``time - time_range`` and ``time`` are considered.
:type time_range: ~datetime.timedelta
:returns: A QuerySet of :class:`~models.InstanceStatus` objects, or a single
:class:`~utilities.DummyInstanceStatus` object when no status is available
in the requested time range.
:rtype: QuerySet or DummyInstanceStatus
"""
from .models import InstanceStatus
# callers without an explicit range (e.g. builddiv passes the legacy
# default ``1``) get the module's own data cadence; the overview
# passes a real timedelta and gets the full requested range
if (
not isinstance(time_range, timedelta)
or time_range <= settings.PULL_INTERVAL
):
time_range = settings.PULL_INTERVAL * instance.get("get_data_every", 1)
else:
# look back one fetch period deeper than requested: the row just
# before the window start (e.g. the hourly SAM row a few seconds
# below the 24 h edge) must be inside the query, so the head
# slots of the overview grid can be bridged by nearest-slot
# mapping or carry-over instead of greying out
time_range += settings.PULL_INTERVAL * instance.get("get_data_every", 1)
return InstanceStatus.objects.get_instance_status(
time=time,
instance_name=instance["name"],
category=instance["category"],
time_range=time_range,
)
[docs]
def get_ajax_endpoints(self, instance, timestamp):
"""
Optional method for analysis modules to declare their AJAX endpoints for cache warmup.
Override this method in your analysis config to enable AJAX endpoint caching during
cache warmup. This prevents empty plots/tables during getData execution.
:param instance: The dict with the properties of an analysis instance.
:type instance: dict
:param timestamp: Unix timestamp to use for time_of_readout parameter.
:type timestamp: int
:returns: List of dicts, each containing:
- 'url_name': Django URL pattern name (e.g., 'dcache_transfer_get_data')
- 'url_kwargs': Dict of URL kwargs (e.g., {'instance_name': 'dCache_Host'})
- 'query_params': Dict of query parameters (e.g., {'time_of_readout': 123, 'dt': 86400})
Returns empty list by default (no AJAX endpoints).
:rtype: list
Example implementation in dcache_transfer/apps.py:
.. code-block:: python
def get_ajax_endpoints(self, instance, timestamp):
return [
{
'url_name': 'dcache_transfer_get_data',
'url_kwargs': {'instance_name': instance['name']},
'query_params': {
'time_of_readout': timestamp,
'dt': instance.get('time_window', 86400)
}
}
]
"""
return [] # Default: no AJAX endpoints