# $SnapHashLicense:
#
# SnapLogic - Open source data services
#
# Copyright (C) 2008 - 2009, SnapLogic, Inc. All rights reserved.
#
# See http://www.snaplogic.org for more information about
# the SnapLogic project.
#
# This program is free software, distributed under the terms of
# the GNU General Public License Version 2. See the LEGAL file
# at the top of the source tree.
#
# "SnapLogic" is a trademark of SnapLogic, Inc.
#
#
# $
# $Id: component_list.py 8047 2009-06-29 17:46:33Z dhiraj $
"""
The component list.
This is a singleton-like object, which holds a compilation of all the
various components that are provided by the different CCs. As such, it
contains methods to populate that list, to query and update the list.
Since multiple requests may arrive from multiple threads, all accesses
to the list are guaranteed to be thread safe.
For each component, we store the name, the name of the CC that provides
it and the URI.
"""
from __future__ import with_statement
from snaplogic import snapi_base,server
from snaplogic.common.config import snap_config
from snaplogic.common import headers,uri_prefix,snap_http_lib
from snaplogic.snapi_base import keys
from snaplogic.common.snap_exceptions import *
from snaplogic.server import RhResponse
from snaplogic.server import product_prefix,cc_list
from snaplogic.server.http_request import HttpRequest
from snaplogic.snapi_base.exceptions import SnapiHttpException
import threading
import copy
from urllib import unquote
__comp_lock = threading.RLock()
__component_list = {}
__map_lock = threading.RLock()
__uri_map = {}
__initial_map = {}
__failed_cc_list = []
def __make_component_list_copy(filter=None):
"""
Make a thread safe copy of the component list.
The component list consists of mostly unmutable objects,
such as strings. So, we can use them by reference. Only
a couple of elements require a deep copy.
This function here knows all that and does the rigt thing.
@param filter: A function that given the name and value of an element
in the per-component dictionary performs some
sort of filtering on that element. If the filter
returns None, then this function here will not copy
the attribute. If no filter is defined (None) then
no filtering at all will take place.
@type filter: callable
@return: A sufficiently deep copy of the component list to
be thread safe.
@rtype: dict
"""
global __comp_lock, __component_list
out = {}
if not filter:
# If no filter was defined, we quickly define it as the identity function
filter = lambda x,y: y
with __comp_lock:
for comp_name in __component_list.keys():
# Now we can process the individual entries, making a copy
# along the way. All the dictionary elements in __component_list
# are strings and thus unmutable. Only 'capabilities' is a bit
# more involved and requires a deep copy.
comp_attribs = __component_list[comp_name]
new_c = {}
for attrib_name in comp_attribs.keys():
v = filter(attrib_name, comp_attribs[attrib_name])
# Only add the component attribute if the filter didn't return None
if v:
if attrib_name is 'capabilities':
new_c[attrib_name] = copy.deepcopy(v)
else:
new_c[attrib_name] = v
# Finally, we can add that new set of component attributes to our copy
out[comp_name] = new_c
return out
def initialize(initial_map = None):
"""
Initialize the URI mapping.
@param initial_map: The caller can provide an initial set of mappings
or None.
@type initial_map: dict
"""
global __component_list, __uri_map, __initial_map
if not initial_map:
__initial_map = {}
else:
__initial_map = initial_map
__uri_map = __initial_map
__component_list = {}
def make_uri_mapping(some_dict, find_str, replace_str):
"""
Fix all URIs that start with CC-specific URI.
We want to shield clients from CC-specific URIs. Therefore, the main
server can perform a simple mapping. All CC-specific URIs are mapped
to a main-server URI.
Two things need to be done: Firstly, we need to fix all occurrences
of the CC-specific URIs. For that, we remove the server-specific portion
at the start of the URI and replace it with our own URI.
Secondly, we need to maintain a mapping dictionary, which allows the
server to quickly perform the necessary lookup and URI re-write. This
dictionary is a module global variable, which we are updating here as
a side effect.
"""
global __uri_map, __map_lock
if not hasattr(some_dict, "keys"):
# Not a dictionary
return
with __map_lock:
# We don't know all the directories and sub-directories in a resdef in
# which URIs may occur. Therefore, we just recursively traverse the entire
# component dictionary and replace all we can.
for i in some_dict.keys():
try:
# Can't be bothered to check the type. If this is NOT a string
# we just silently catch and ignore the exception.
item = some_dict[i]
if not item:
continue
# Now that these URIs are relative, we can't rely on finding the
# CC path to it. So we have to know which ones are URIs...
# There should be a better way of designating them, perhaps?
if i in keys.CAPABILITY_URIS:
cc_path = snap_http_lib.concat_paths(find_str, item)
__uri_map.update( { item : cc_path} )
continue
elif item.startswith(find_str):
path = item[len(find_str):]
some_dict[i] = unquote(replace_str + path)
# We only need the path as the search key, since all requests
# into the main server will have the server's server prefix anyway.
# Nothing will change there...
__uri_map.update( { path : item } )
continue
except Exception, e:
pass
try:
# Dive one recursion level deeper, if this is a dictionary.
# Can't be bothered to check if this is a dictionary or not.
# We just call it and if we then get an exception, we just
# silently catch and ignore it.
make_uri_mapping(item, find_str, replace_str)
except:
pass
def map_uri(uri):
"""
Map the specified URI to the one we have in the map.
This is used when requests for main-server component URIs
come in, which need to be translated into CC-specific URIs.
@param uri: The URI that needs to be translated.
@type uri: string
@return: Mapped URI, or None, if the mapping does not exist.
@rtype string
"""
populate()
with __map_lock:
# Strings are immutable. How convenient: We don't need to make a copy of it.
# get() will return None, if entry was not found.
return __uri_map.get(uri)
def populate():
"""
Populate and update our component list.
This locks down the component list, and queries
CCs for their component list.
Whatever else was stored in our component list before
will be lost, except any initial mappings that were provided
to initialize().
Note also that this will create the mapping between main-server URIs
and CC-specific URIs for components and related actions on those
components. The point here is that we are shielding clients from the
CC-specific URIs. The main-server will take incoming requests and map
them to the specific CC-URIs.
@return: Mapping dictionary, using the main-server URI as key to
look up the CC URI.
@rtype: dict
"""
global __comp_lock, __component_list, __uri_map, __failed_cc_list
with __comp_lock:
if __component_list and not __failed_cc_list:
return
try:
conf = snap_config.get_instance()
cc_conf = conf.get_section("component_container")
main_process_uri = conf.get_section("common")['main_process_uri']
if not __component_list:
# We don't have any component information at this point of time, its a fresh start.
# Query all CCs.
__uri_map = __initial_map
cc_keys = cc_conf.keys()
else:
# Query only the CCs in config that are listed as failed CCs
cc_keys = [cc for cc in cc_conf.keys() if cc in __failed_cc_list]
failures = []
for cc in cc_keys:
cc_uri = cc_list.cc_map[cc]['uri']
try:
comp_dict = snapi_base.list_components(cc_uri)
for comp_name in comp_dict.keys():
# Once we get the dictionary that represents a component's resdef
# we need to crawl through the thing and find all CC-specific
# URIs. We replace them with mapped URIs, that point to something
# in the main server.
make_uri_mapping(comp_dict[comp_name], cc_uri, main_process_uri)
# Patchup the dictionary for missing URIs.
_add_missing_capability_uris(comp_name, comp_dict, main_process_uri)
# Lastly, we add something to each component's resdef. For example,
# the 'server URI' of that component's CC. We do that after we mapped
# all the other URIs, because we don't want to replace that URI.
comp_dict[comp_name]['uri'] = cc_uri
__component_list.update(comp_dict)
except Exception, e:
# Just log an exception and move on to the next CC. Make note of the failed CC,
# because we will continue to retry it in future requests, until we get a response
# from it.
server.elog(e, "Cannot communicate with CC '%s' to accumulate list of components." % cc)
failures.append(cc)
except Exception, e:
raise SnapException.chain(e, SnapGeneralError("Cannot accumulate list of components from CCs."))
__failed_cc_list = failures
def _add_missing_capability_uris(comp_name, comp_dict, main_process_uri):
"""
For create_resource_template/suggest_resource_values/validate point to server, if CC does not provide URIs.
If the capabilities dictionary has no URI for create_resource_template/suggest_resource_values/validate
(because the component does not implement the methods), then provide 'default' URIs that point to the
server. The requests for 'default' URIs will endup in the server and since no corresponding entry exists
in the URI mapping dictionary, the server will endup handling the request, instead of forwarding them to
the CC.
The comp_dict dictionary will be modified by this method, so the method should be called after appropriate
locks have been called.
@param comp_name: Name of the component
@type comp_name: str
@param comp_dict: The dictionary for the component, obtained from the list components.
@type comp_dict: dict
@param main_process_uri: The URI of this server process.
@type main_process_uri: str
"""
capabilities = comp_dict[comp_name]['capabilities']
for (c, prefix) in ((keys.CAPABILITY_CREATE_RESOURCE_TEMPLATE, uri_prefix.COMPONENT_RESOURCE_TEMPLATE),
(keys.CAPABILITY_SUGGEST_RESOURCE_VALUES, uri_prefix.COMPONENT_SUGGEST_RESOURCE_VALUES),
(keys.CAPABILITY_VALIDATE, uri_prefix.COMPONENT_VALIDATE_RESOURCE)):
u = capabilities.get(c)
if u is None:
capabilities[c] = snap_http_lib.concat_paths(prefix, comp_name)
def get_cc_uri_for_component(comp_name):
"""
Return the URI of the CC in which a given component is located.
This returns only the root URI of the CC server process, nothing
more.
@param comp_name: The name of the component, for example:
'snaplogic.components.CsvWrite'.
@type comp_name: string
@return: The root URI of the CC server process.
@rtype: string
"""
global __comp_lock, __component_list
populate()
with __comp_lock:
try:
uri = __component_list[comp_name]['uri']
except Exception, e:
raise SnapException.chain(e, SnapComponentError("Cannot find component '%s'." % (comp_name)))
return uri
def get_capabilities_for_component(comp_name):
"""
Return the capabilities of the component .
@param comp_name: The name of the component, for example:
'snaplogic.components.CsvWrite'.
@type comp_name: string
@return: The capabilities dict of the component
@rtype: dict
"""
global __comp_lock, __component_list
populate()
with __comp_lock:
try:
capabilities = copy.deepcopy(__component_list[comp_name]['capabilities'])
except Exception, e:
raise SnapException.chain(e, SnapComponentError("Cannot find component '%s'." % (comp_name)))
return capabilities
def get_component_info(comp_name):
"""
Return the information in component list about the specified component .
@param comp_name: The name of the component, for example:
'snaplogic.components.CsvWrite'.
@type comp_name: string
@return: The info dict of the component
@rtype: dict
"""
global __comp_lock, __component_list
populate()
with __comp_lock:
try:
cinfo = copy.deepcopy(__component_list[comp_name])
except Exception, e:
raise SnapException.chain(e, SnapComponentError("Cannot find component '%s'." % (comp_name)))
return cinfo
def rh_get_component_list(http_req):
"""
Get the combined list of components.
This function is part of the RH front end of this module. It can
be directly used as a module specific request handler.
We don't want to hold the lock while we output the information, and we
also want to censor the information a little bit. That's why we make a
copy, with some additional processing on it. This has to be a deep copy,
though...
@param http_req: The HTTP request object, passed in by the RH.
The function doesn't need this object, but it is
presented by the RH to all request handlers.
@type http_req: L{HttpReq}
@return: RhResponse object with data and code
to be written to client.
@rtype: L{RhResponse}
"""
populate()
# Here we create a the component list, but filter out all occurrences
# of the URI of the component's server, since we want to shield users
# from the fact that there are CCs.
out = __make_component_list_copy(lambda x, y: y if x != 'uri' else None)
return RhResponse(200, out, None, { 'title' : product_prefix.SNAPLOGIC_PRODUCT + ': Component list' })
def validate_with_cc(comp_name, resdef_dict):
"""
Makes a resdef validation request to the CC with a specific component name
This method locates the appropriate CC which hosts the component and
makes a network request to validate the resdef_dict that has been
provided. If the resdef is valid, then None is returned. If not, an
L{snapi_base.ValidationDict} is returned.
@param comp_name: Name of the component that should perform the
validation.
@type comp_name: string
@param resdef_dict: The resource definition dictionary that should be
validated.
@type resdef_dict: dict
@return: A dictionary with the validation results (which
may contain possible error messages).
@rtype: L{snapi_base.ValidationDict}
"""
try:
capabilities = get_capabilities_for_component(comp_name)
conf = snap_config.get_instance()
main_process_uri = conf.get_section("common")['main_process_uri']
if 'validate' not in capabilities or not capabilities['validate']:
# This component has no custom validation.
# We should do some basic structural tests.
return None
comp_uri = capabilities['validate']
if comp_uri.startswith(main_process_uri):
comp_uri = comp_uri[len(main_process_uri):]
comp_uri = map_uri(comp_uri)
if comp_uri is None:
# The component has not implemented this method. Return None.
return None
except Exception, e:
raise SnapException.chain(e, SnapObjNotFoundError("Failed to get capabilities of component %s: " % comp_name))
# Fetch the token for the CC.
cc_token = cc_list.get_token_by_uri(comp_uri)
if cc_token is None:
raise SnapObjNotFoundError("No CC token found for URI %s" % comp_uri)
try:
snapi_base.send_req("POST", comp_uri, resdef_dict, {headers.CC_TOKEN_HEADER: cc_token})
except SnapiHttpException, e:
if e.status == HttpRequest.BAD_REQUEST:
return snapi_base.ValidationDict(e.body)
else:
raise
return None
|