You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
447 lines
16 KiB
447 lines
16 KiB
# Copyright (C) 2005-2024 Splunk Inc. All Rights Reserved.
|
|
|
|
import copy
|
|
import time
|
|
|
|
from itsi_py3 import _
|
|
from ITOA.itoa_common import is_valid_dict, is_valid_list, is_valid_str
|
|
from ITOA.storage.itoa_storage import ITOAStorage
|
|
from ITOA.setup_logging import getLogger
|
|
|
|
|
|
class ItoaGenericCrudException(Exception):
|
|
pass
|
|
|
|
|
|
class ItoaStorageInterfaceAdapter(object):
|
|
"""
|
|
State store interface adapter
|
|
|
|
Proxies the state store CRUD operations. Most of the time, returns
|
|
the statestore response directly; for the `get` method, normalize the
|
|
response to always be a list.
|
|
"""
|
|
|
|
def __init__(self, session_key, collection_name, owner="nobody", namespace=None, object_type=None, **kwargs):
|
|
"""
|
|
@param session_key: splunkd session key
|
|
@param collection_name: collection name as configured in collections.conf
|
|
@param owner: owner (defaults to `nobody`)
|
|
@param namespace: app namespace where the collection is defined
|
|
@param object_type: [optional] ITSI object type; ITOA kv crud methods expect this argument; defaults to `itoa_generic_object`
|
|
@param **kwargs: arguments to pass on to itoa_storage initializer; allowed args: `splunkd_host_path`, `splunkd_port`
|
|
"""
|
|
self._session_key = session_key
|
|
self._owner = owner
|
|
self._object_type = object_type or "itoa_generic_object"
|
|
self._itoa_storage = ITOAStorage(collection=collection_name, namespace=namespace, **kwargs)
|
|
if not self._itoa_storage.wait_for_storage_init(self._session_key):
|
|
raise ItoaGenericCrudException(_("KV store is unavailable."))
|
|
|
|
def create(self, data):
|
|
"""
|
|
@param data: object to write to the statestore
|
|
@type: dict
|
|
"""
|
|
return self._itoa_storage.create(self._session_key, self._owner, self._object_type, data)
|
|
|
|
def get(self, identifier=None, filter_data=None, **kwargs):
|
|
"""
|
|
Proxies state store object retrieval; ensures the result is always a
|
|
list of dictionaries. All arguments are optional; empty arg list results in
|
|
fetching the full collection.
|
|
|
|
@param identifier: _key (if retrieving object by ID)
|
|
@type: string
|
|
|
|
@param filter_data: filter data object (if retrieving a set of objects)
|
|
@type: dict
|
|
|
|
@rtype: list
|
|
@returns list of matched objects from the state store if using filter_data, or a
|
|
single-element list if querying by id
|
|
"""
|
|
if identifier is not None:
|
|
response = self._itoa_storage.get(self._session_key, self._owner, self._object_type, identifier)
|
|
else:
|
|
response = self._itoa_storage.get_all(self._session_key, self._owner, self._object_type, filter_data=filter_data, **kwargs)
|
|
if is_valid_dict(response):
|
|
return [response] if len(response) > 0 else []
|
|
elif response is None:
|
|
return []
|
|
elif is_valid_list(response):
|
|
return response
|
|
else:
|
|
raise ItoaGenericCrudException(_("Invalid response received from the backend."))
|
|
|
|
def edit(self, identifier, data):
|
|
"""
|
|
@param identifier: _key of the object to edit
|
|
@type: string
|
|
|
|
@param data: edit data
|
|
@type: dict
|
|
"""
|
|
return self._itoa_storage.edit(self._session_key, self._owner, self._object_type, identifier, data)
|
|
|
|
def delete(self, identifier):
|
|
"""
|
|
@param identifier: _key of the object to delete
|
|
@type: string
|
|
"""
|
|
return self._itoa_storage.delete(self._session_key, self._owner, self._object_type, identifier)
|
|
|
|
|
|
class ItoaGenericPersistableBase(object):
|
|
"""
|
|
Abstract base class for an ITOA persistable object.
|
|
"""
|
|
backing_collection = None
|
|
object_type = "itoa_generic_object"
|
|
|
|
def __init__(self, logger=None, interface=None, session_key=None, owner="nobody", namespace=None, **kwargs):
|
|
"""
|
|
@param logger [optional]: class logger; defaults to a dummy logger
|
|
@type: logging.logger
|
|
|
|
@param interface: interface reference
|
|
@type: ItoaStorageInterfaceAdapter
|
|
|
|
@param session_key: splunkd session key
|
|
@type: string
|
|
|
|
@param owner [optional]: owner; defaults to `nobody`
|
|
@type: string
|
|
|
|
@param namespace [optional]: app namespace; default defined in itoa_storage interface
|
|
@type: string
|
|
|
|
@param **kwargs [optional]: optional kwargs passed down to storage interface
|
|
"""
|
|
if self.backing_collection is None:
|
|
raise ItoaGenericCrudException(_("KV store collection name must be supplied as a class variable."))
|
|
if logger is not None:
|
|
self.logger = logger
|
|
elif not hasattr(self, 'logger'): # in case a subclass initializer already gave us a logger
|
|
self.logger = getLogger(logger_name='__DUMMY')
|
|
# Initialize the storage interface
|
|
if interface is not None:
|
|
self._interface = interface
|
|
elif session_key is not None and owner is not None:
|
|
self._interface = ItoaStorageInterfaceAdapter(session_key, self.backing_collection,
|
|
owner=owner, object_type=self.object_type, namespace=namespace,
|
|
**kwargs)
|
|
else:
|
|
raise ItoaGenericCrudException(_("Either interface or session key/owner must be provided."))
|
|
|
|
@property
|
|
def interface(self):
|
|
return self._interface
|
|
|
|
@classmethod
|
|
def initialize_interface(cls, session_key, owner="nobody", namespace=None, **kwargs):
|
|
"""
|
|
Class-level method to instantiate `ItoaStorageInterfaceAdapter` class for a given KV store collection.
|
|
The returned object can be used when initializing `ItoaGenericModel` and `ItoaGenericCollection` objects.
|
|
This is more efficient than initializing those objects from a sesion key, which always creates a new instance
|
|
of `ItoaStorageInterfaceAdapter`.
|
|
|
|
@param session_key: splunkd session key
|
|
@type: string
|
|
|
|
@param owner [optional]: owner; defaults to `nobody`
|
|
@type: string
|
|
|
|
@returns an instance of `ItoaStorageInterfaceAdapter`
|
|
@rvalue: `ItoaStorageInterfaceAdapter`
|
|
"""
|
|
return ItoaStorageInterfaceAdapter(session_key, cls.backing_collection, owner, namespace, cls.object_type, **kwargs)
|
|
|
|
def set_logger(self, logger):
|
|
"""
|
|
@param logger: logger to use for this class
|
|
@type: logging.logger
|
|
"""
|
|
self.logger = logger
|
|
|
|
def fetch(self):
|
|
raise NotImplementedError(_("Attempting to call an abstract method"))
|
|
|
|
def save(self):
|
|
raise NotImplementedError(_("Attempting to call an abstract method"))
|
|
|
|
|
|
class ItoaGenericModel(ItoaGenericPersistableBase):
|
|
"""
|
|
Itoa generic persistable model class. Can be instantiated from data, or from an object
|
|
_key (in which case it is auto-fetched from the server). Supports basic CRUD
|
|
operations.
|
|
"""
|
|
|
|
def __init__(self, data, key=None, collection=None, _fetch=False, **kwargs):
|
|
"""
|
|
Either `ItoaStorageInterfaceAdapter` reference or session_key/owner
|
|
params must be supplied on instantiation.
|
|
|
|
@param data: dict of parameters for this model
|
|
@type: dict
|
|
|
|
@param key [optional]: custom key if one must be specified on object creation.
|
|
DO NOT use`data._key` for this!
|
|
@type: string
|
|
|
|
@param logger [optional]: class logger
|
|
@type: logging.logger
|
|
|
|
@param interface: interface reference
|
|
@type: ItoaStorageInterfaceAdapter
|
|
|
|
@param session_key: splunkd session key
|
|
@type: string
|
|
|
|
@param owner [optional]: owner; defaults to `nobody`
|
|
@type: string
|
|
|
|
@param collection: collection to associate with this model. If this
|
|
parameter is specified, deletions result in removal of this model reference
|
|
from the collection.
|
|
@type: ItoaGenericCollection
|
|
|
|
@param _fetch: auto-fetch flag; used internally by alternative constructors
|
|
@type: bool
|
|
"""
|
|
super(ItoaGenericModel, self).__init__(**kwargs)
|
|
self.data = copy.copy(data)
|
|
self._user_supplied_key = key
|
|
self._collection = collection
|
|
if _fetch:
|
|
self.fetch()
|
|
|
|
@classmethod
|
|
def fetch_from_key(cls, key, collection=None, **kwargs):
|
|
"""
|
|
Constructor that allows to construct a model object from _key alone;
|
|
fetches the object from the backend automatically.
|
|
|
|
@param key: object id
|
|
@type: string
|
|
|
|
@param interface: interface reference
|
|
@type: ItoaStorageInterfaceAdapter
|
|
|
|
@param session_key: splunkd session key
|
|
@type: string
|
|
|
|
@param owner: owner (optional; defaults to `nobody`)
|
|
@type: string
|
|
|
|
@param collection: collection to associate with this model. If this
|
|
parameter is specified, deletions result in removal of this model reference
|
|
from the collection.
|
|
@type: ItoaGenericCollection
|
|
"""
|
|
return cls({}, key=key, collection=collection, _fetch=True, **kwargs)
|
|
|
|
def validate_data(self):
|
|
"""
|
|
User-supplied validation method. Default one always returns `True`
|
|
Validation runs on both fetch and create (possibly before the model is persisted
|
|
to the server, thus there should be no assumptions about the presence of server-
|
|
generated fields).
|
|
"""
|
|
return True
|
|
|
|
@property
|
|
def is_new(self):
|
|
"""
|
|
The object is defined to be new if it contains the `_key` field. If the key is specified
|
|
but the model with this key does not actually exist on the server, CRUD operations will fail.
|
|
|
|
@returns is_new status
|
|
@rtype: bool
|
|
"""
|
|
return '_key' not in self.data or not is_valid_str(self.data['_key'])
|
|
|
|
def _create(self):
|
|
if not self.validate_data():
|
|
raise ItoaGenericCrudException(_("Failed to create model: data validation failed."))
|
|
# explicitly set the _key field if the user chooses to do so
|
|
if self._user_supplied_key:
|
|
self.data["_key"] = self._user_supplied_key
|
|
response = self._interface.create(self.data)
|
|
if response and "_key" in response:
|
|
self.data.update(response)
|
|
return self.data
|
|
else:
|
|
raise ItoaGenericCrudException(_("Failed to create model."))
|
|
|
|
def _update(self):
|
|
response = self._interface.edit(self.data["_key"], self.data)
|
|
if not (is_valid_dict(response) and "_key" in response):
|
|
raise ItoaGenericCrudException(_("Failed to update model."))
|
|
self.data.update(response)
|
|
return self.data
|
|
|
|
def update(self, data):
|
|
"""
|
|
Updates the model object and saves it.
|
|
|
|
@param data: dict of KV pairs to update the model with
|
|
@type: dict
|
|
|
|
@returns an updated model object
|
|
@rtype: dict
|
|
"""
|
|
self.data.update(data)
|
|
return self.save()
|
|
|
|
def save(self):
|
|
"""
|
|
Persists the model object to the server.
|
|
|
|
@returns the model object as a dict (including the _key returned by the server)
|
|
@rtype: dict
|
|
"""
|
|
try:
|
|
if self.is_new:
|
|
return self._create()
|
|
else:
|
|
return self._update()
|
|
except Exception:
|
|
self.logger.exception("Error saving model")
|
|
raise ItoaGenericCrudException(_("Failed to save model."))
|
|
|
|
def _ensure_key(self):
|
|
"""
|
|
If this method returns True, a valid self.data["_key"] is now present.
|
|
This method may be called from within CRUD operations in order to fall back on
|
|
user-supplied key in case data["_key"] is msising.
|
|
"""
|
|
if self.is_new and is_valid_str(self._user_supplied_key):
|
|
self.data["_key"] = self._user_supplied_key
|
|
return is_valid_str(self.data.get("_key", None))
|
|
|
|
def fetch(self):
|
|
"""
|
|
Fetches the model object from the server; updates internal state.
|
|
|
|
@returns the model object as a dict
|
|
@rtype: dict
|
|
"""
|
|
try:
|
|
if not self._ensure_key():
|
|
raise ItoaGenericCrudException(_("Cannot fetch an unsaved model."))
|
|
|
|
response = self._interface.get(self.data["_key"])
|
|
assert len(response) > 0, "Empty response received"
|
|
assert "_key" in response[0], "Response missing _key field"
|
|
self.data.update(response[0])
|
|
assert self.validate_data(), "Failed to validate fetched response"
|
|
return self.data
|
|
except Exception as e:
|
|
self.logger.exception("Error fetching model")
|
|
raise ItoaGenericCrudException(_("Failed to fetch model: %s.") % e)
|
|
|
|
def delete(self):
|
|
"""
|
|
Removes the model from the server; if this model is associated
|
|
with a collection, this object's reference is located in the collection and
|
|
deleted.
|
|
"""
|
|
try:
|
|
if not self._ensure_key():
|
|
self.logger.debug("Ignoring the delete() call for unsaved model with no user-provided keys")
|
|
return
|
|
|
|
res = self._interface.delete(self.data["_key"])
|
|
if self._collection: # remove this model's reference
|
|
idx = next((i for i, x in enumerate(self._collection) if x.data.get("_key", None) == self.data["_key"]), None)
|
|
if idx is not None:
|
|
del self._collection[idx]
|
|
return res
|
|
except Exception:
|
|
self.logger.exception("Error deleting model")
|
|
raise ItoaGenericCrudException(_("Failed to delete model."))
|
|
|
|
|
|
class ItoaGenericCollection(ItoaGenericPersistableBase):
|
|
"""
|
|
Generic collection class. Supports bulk fetch, save, and delete
|
|
operations. Implements an iterable interface over `ItoaGenericModel` objects.
|
|
"""
|
|
|
|
model_class = ItoaGenericModel
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
"""
|
|
Either `ItoaStorageInterfaceAdapter` reference or session_key/owner
|
|
params must be supplied on instantiation.
|
|
|
|
@param logger: class logger
|
|
@type: logging.logger
|
|
|
|
@param interface: interface reference
|
|
@type: ItoaStorageInterfaceAdapter
|
|
|
|
@param session_key: splunkd session key
|
|
@type: string
|
|
|
|
@param owner [optional]: owner; defaults to `nobody`
|
|
@type: string
|
|
"""
|
|
super(ItoaGenericCollection, self).__init__(*args, **kwargs)
|
|
self._data = []
|
|
|
|
def fetch(self, filters=None):
|
|
"""
|
|
Fetches a set of model objects from the backend and stores them
|
|
internally.
|
|
|
|
@param filters: optional filterspec; if empty returns all objects
|
|
@type: dict
|
|
|
|
@returns: list of `ItoaGenericModel` objects (that is also stored internally)
|
|
@rtype: list
|
|
"""
|
|
try:
|
|
response = self._interface.get(filter_data=filters)
|
|
self._data = [self.model_class(
|
|
data, interface=self._interface, collection=self
|
|
) for data in response]
|
|
assert all(x.validate_data() for x in self._data), "Failed to validate data for some models"
|
|
return self._data
|
|
except Exception as exc:
|
|
self.logger.exception("Error fetching collection")
|
|
raise ItoaGenericCrudException(_("Failed to fetch collection: %s.") % exc)
|
|
|
|
def save(self):
|
|
"""
|
|
Iterates over the `ItoaGenericModel` objects saving them individually.
|
|
"""
|
|
for req in self._data:
|
|
req.save()
|
|
|
|
def delete(self):
|
|
"""
|
|
Iterates over the `ItoaGenericModel` objects deleting them.
|
|
"""
|
|
# delete() on a model also deletes its reference from the
|
|
# internal _data array, so it's only safe to iterate in reverse
|
|
for i in range(len(self._data) - 1, -1, -1):
|
|
self._data[i].delete()
|
|
|
|
@property
|
|
def models(self):
|
|
"Public accessor to the internal array of `ItoaGenericModel`s"
|
|
return self._data
|
|
|
|
def __len__(self):
|
|
return len(self._data)
|
|
|
|
def __getitem__(self, i):
|
|
return self._data[i]
|
|
|
|
def __delitem__(self, i):
|
|
"Removes model reference from the collection WITHOUT removing it from the server"
|
|
return self._data.__delitem__(i)
|