#!/usr/bin/env python
# $SnapHashLicense:
#
# SnapLogic - Open source data services
#
# Copyright (C) 2008-2009, SnapLogic, Inc. All rights reserved.
#
# See http://www.snaplogic.org for more information about
# the SnapLogic project.
#
# This program is free software, distributed under the terms of
# the GNU General Public License Version 2. See the LEGAL file
# at the top of the source tree.
#
# "SnapLogic" is a trademark of SnapLogic, Inc.
#
#
# $
from __future__ import with_statement
from cmd import Cmd
import getpass
import logging
import os
import os.path
import re
import getopt
import sys
import traceback
from stat import *
from optparse import OptionParser,IndentedHelpFormatter
import urlparse
import pprint
import fnmatch
import simplejson
import time
import string
import random
import urllib
import textwrap
# readline is an optional package, so ignore inmport errors
try:
import readline
except ImportError:
pass
import snaplogic.common.config.snap_config as snap_config
from snaplogic.common.config.snap_config import SnapConfig
from snaplogic.common.snap_crypt import md5crypt,make_salt
from snaplogic.common.snap_exceptions import *
from snaplogic.common import file_lock
from snaplogic.common import uri_utils
from snaplogic.common import snap_http_lib
from snaplogic.common.runtime_status import RuntimeStatus
from snaplogic.server.http_request import HttpRequest
from snaplogic.server import repository
from snaplogic.server.repository.packaging import PackageReader,PackageWriter
from snaplogic import snapi_base
from snaplogic.snapi_base.exceptions import SnapiException,SnapiHttpException
from snaplogic import snapi
from snaplogic.snapi_base import keys
from snaplogic.snapi_base import resdef
from snaplogic.snapi import exec_resource
from snaplogic.snapi_base.exec_interface import get_status,send_stop
verbose_mode = False
""" Verbose controls how much detail is printed when an error occurs """
verbose_lasterr = None
""" Additional information about the last error occurred (printed if "verbose lasterr" is requested) """
class UriStyle:
ABSOLUTE = "absolute"
RELATIVE = "relative"
class CmdSyntaxError(Exception):
pass
def printerr(*args):
msg = list()
msg.append("Problem: ")
for arg in args:
msg.append(arg)
sys.stderr.write(''.join(msg) + "\n")
def printexc(e):
""" Report on exception. Amount of detail printed depends on verbose_mode setting. """
global verbose_mode, verbose_lasterr
if e.message:
printerr(e.message)
else:
printerr(str(e))
if verbose_mode:
# Print stack trace if in verbose mode
traceback.print_exc(file=sys.stderr)
# Save stacktrace: if "verbose lasterr" is called we'll print it.
verbose_lasterr = traceback.format_exc()
print ""
print "To display additional error information type"
print "verbose lasterr"
def printok(*args):
msg = list()
msg.append("Success: ")
for arg in args:
msg.append(arg)
print ''.join(msg)
def confirmed():
"""
Confirm prompt
"""
sys.stdout.write("Are you sure? (Y/N) ")
ans = sys.stdin.readline()
return not (len(ans) == 0 or str.lower(ans)[0] != 'y')
def batch_commands(batch_file, process=None, quiet_mode=False):
"""
Execute snapadmin commands from a batchfile
The 'command processing function' is a function that takes a
snapadmin command as a string, and parses and executes the
command.
@param batch_file : Path to file containing commands to execute
@type batch_file : string
@param process : Command processing function
@type process : function: string -> None
"""
if process is None:
process = processor.onecmd
f = open(batch_file, 'r')
cmds = f.readlines()
for cmd in cmds:
cmd = cmd.strip()
if len(cmd):
if not quiet_mode:
print "* Executing " + cmd + " ..."
process(cmd)
print ""
def printdata(data, start_indent=0, out=sys.stdout):
"""
Print out an arbitrary data structure
@param data : Data structure to print out
@type data : Pretty much any Python data type
@param start_indent : Starting indentation level, in spaces
@type start_indent : int
@param out : File to write output to
@type out : file
"""
from collections import defaultdict
INDENT_STEP = 2 # how many spaces to indent each level
# some utility functions for printing out different types
def pr_other(item, indent, out):
out.write("%s%s\n" % (' ' * indent, str(item)))
def pr_list(items, indent, out):
for val in items:
prmap[type(val)](val, indent, out)
def pr_dict(item, indent, out):
for kk, vv in item.iteritems():
out.write("%s%s:\n" % (' ' * indent, kk))
prmap[type(vv)](vv, indent+INDENT_STEP, out)
# table mapping types to their printer functions
prmap = defaultdict(lambda: pr_other)
prmap.update({
dict : pr_dict,
list : pr_list,
tuple: pr_list,
})
prmap[type(data)](data, start_indent, out)
def _print_upgrade_result(upgrade_result, always_show=False):
had_error = False
upgraded_count = 0
for upgrade_info in upgrade_result.itervalues():
if upgrade_info['upgraded']:
upgraded_count += 1
if 'error' in upgrade_info:
had_error = True
if upgraded_count > 0 or had_error or always_show:
print "Upgraded %d resources." % upgraded_count
if had_error:
print "Failed to upgrade the following resources:"
for (uri, upgrade_info) in upgrade_result.iteritems():
if 'error' in upgrade_info:
print " %s: %s" % (uri, upgrade_info['error'])
# Globals used in all commands
processor = None
class ConnectionResources(object):
"""
Manages global variables and server-connection resources
"""
#: Index of current connection within self.connections. Set to None if no connections
conn_index = None
#: Default credentials
default_creds = None
class ConnectType:
direct = 'direct'
server = 'server'
current = None
def __init__(self):
self.connections = []
def connect(self, server_url):
info = snapi_base.get_server_info(server_url, cred=self.get_creds())
if (type(info) is not dict) or ('server_version' not in info):
printerr("connect server failed: invalid server info received.")
return
self.connections.append({'uri' : server_url,
'creds' : None,
})
self.conn_index = len(self.connections) - 1
printok("Connected to server.")
print " Server URL: %s" % server_url
print " Server version:", info['server_version']
print " Server copyright:", info['server_copyright']
def disconnect(self):
"""
Disconnect from the current connection
"""
removed = self.connections.pop(self.conn_index)
if self.connections:
self.conn_index = 0
printok("Now connected to %s." % self.server_url)
else:
self.conn_index = None
def get_runtime_status(self):
"""
Get all runtime status information available on the server
(this is similar to visiting the http://localhost:8088/__snap__/runtime/status page)
"""
return snapi_base.get_runtime_status(self.server_url, cred = self.get_creds())
def get_pipelines(self):
"""
Get all pipelines in the current server connection
@return : mapping of pipeline URI(s) to an info dict
@rtype : dict
"""
def is_pipeline(info):
# Is there a more reliable test than this?
return snapi_base.keys.VIEW_LINKS in info['resdef']
resources = snapi_base.read_resources(snapi_base.list_resources(self.server_url).keys(), cred=self.get_creds())
if resources is None:
return {}
return dict((uri, info)
for uri, info in \
resources['success'].iteritems()
if is_pipeline(info))
# Much code and tests use the old, obsolete server_url attribute.
# Create a property that mimics it so we don't have to refactor
# all that code yet.
def _set_server_url(self, url):
if self.conn_index is None:
printerr("No connection, cannot set server URI.")
else:
self.connections[self.conn_index]['uri'] = url
def _get_server_url(self):
if self.conn_index is None:
return None
return self.connections[self.conn_index]['uri']
server_url = property(fget=_get_server_url, fset=_set_server_url)
def set_default_creds(self, user, passwd):
"""
Set default connection credentials
@param user : user name
@type user : string
@param passwd : password
@type passwd : string
"""
self.default_creds = (user, passwd)
# Loop through all the connections and validate credentials
# using the default ones specified. This only applies to the connections
# that don't have explicit credentials associated with them.
for conn in self.connections:
if conn.get('creds') is None:
self.validate_creds({'uri' : conn['uri'], 'creds' : self.default_creds})
def set_creds(self, user, passwd, index=None):
"""
Set credentials for a specific connection
If index is None, the current connection's credentials will be set.
@param user : user name
@type user : string
@param passwd : password
@type passwd : string
@param index : Index of connection to change, if multiple connections are set
@type index : int, or None
@raise IndexError The index is outside the range of current connections
"""
if index is None:
index = self.conn_index
conn = self.connections[index]
conn['creds'] = (user, passwd)
# Validate the credentials
self.validate_creds(conn)
def validate_creds(self, conn):
""" Validate the credentials """
try:
snapi.user_auth_check(conn['uri'], conn['creds'])
except Exception:
printerr("Cannot connect to %s" % conn['uri'])
# Parent's exception handler will print more detail about this exception
raise
def reset_creds(self, index=None):
"""
reset (clear) credentials
If index is None, the current connection's credentials will be reset.
@param index : Index of connection to change, if multiple connections are set
@type index : int, or None
@raise IndexError The index is outside the range of current connections
"""
if index is None:
if self.conn_index is None:
return
index = self.conn_index
self.connections[index]['creds'] = None
def reset_default_creds(self):
"""
reset (clear) default credentials
@param index : Index of connection to change, if multiple connections are set
@type index : int, or None
"""
self.default_creds = None
def get_creds(self):
"""
Get credentials for current connection
Fall back on default credentials if nothing specific has been
set for the current connection or if we don't have a connection
at this point.
"""
try:
creds = self.connections[self.conn_index]['creds']
except:
creds = None
if creds is None:
return self.default_creds
return creds
def switch(self, index):
"""
Switch to a different server connection
@param index : Index of connection to switch to
@type index : int
"""
if index < 0 or index >= len(self.connections):
printerr("connection id out of range>")
return
self.conn_index = index
def is_connected(self):
"""
Tell whether we are currently connected to a server
@return : True iff we have at least one connection active
@rtype : bool
"""
return self.server_url is not None
connres = ConnectionResources()
TOKEN_SPLIT_RE = re.compile(r"""(\s+|\\.?|'|\")""")
def decode_backslash(string):
if string and string[0] != '\\':
return string
elif len(string) == 2:
return string[1]
else:
raise SnapValueError("Character required after '\\'.")
def tokenize(string):
"""
@param string : command line argument string
@type string : string
@return : tokens
@rtype : list of strings
"""
OUT_OF_TOKEN = 'out of token'
IN_TOKEN = 'in token'
IN_SINGLE_QUOTE = 'in single quote'
IN_DOUBLE_QUOTE = 'in double quote'
splits = TOKEN_SPLIT_RE.split(string)
tokens = []
state = OUT_OF_TOKEN
for part in splits:
if not part:
# Two delimiters were next to eachother
continue
elif state == OUT_OF_TOKEN:
if part.isspace():
continue
else:
state = IN_TOKEN
tokens.append('')
if state == IN_TOKEN:
if part.isspace():
state = OUT_OF_TOKEN
elif part[0] == '\\':
tokens[-1] += decode_backslash(part)
elif part[0] == "'":
state = IN_SINGLE_QUOTE
elif part[0] == '"':
state = IN_DOUBLE_QUOTE
else:
tokens[-1] += part
elif state == IN_SINGLE_QUOTE:
if part[0] == "'":
state = IN_TOKEN
else:
tokens[-1] += decode_backslash(part)
elif state == IN_DOUBLE_QUOTE:
if part[0] == '"':
state = IN_TOKEN
else:
tokens[-1] += decode_backslash(part)
else:
raise SnapGeneralError("Invalid parser state (%s)." % state)
return tokens
def convert_uri_list(uri_list, uri_style):
"""
Convert a list of URIs between relative and absolute.
@param uri_list: A list of URI strings.
@type uri_list: list
@param uri_style: Style of URI to return. All URIs will be converted if necessary.
@type uri_style: str
@return: List of absolute or relative URIs.
@rtype: list
"""
converted = []
for uri in uri_list:
if uri_style is UriStyle.ABSOLUTE:
if uri.startswith('/'):
uri = connres.server_url + uri
else:
if not uri.startswith('/'):
uri = snap_http_lib.parse_uri(uri).path
converted.append(uri)
return converted
def match_server_uris(pattern_list, recursive=False, uri_style=UriStyle.RELATIVE):
"""
Match a list of URI path patterns against the server URI list.
Using SnAPI, the list of resources is retrieved via list_resources(). Then each
URI pattern in pattern_list is matched against the list, and the unique set of
matching URIs for all patterns is returned as a list.
If the recursive flag is given and True, patterns that match folders along the path
will also match all URIs with that folder as a prefix.
If uri_style is given, it specifies the style of URI to return.
See L{snaplogic.common.uri_utils.path_glob_match} for more information about
URI pattern matching.
@param pattern_list: A list of URI path pattern globs.
@type pattern_list: list
@param recursive: A flag indicating if recursive pattern matching should be used.
@type recursive: bool
@param uri_style: Style of URI to return. All URIs will be converted if necessary.
@type uri_style: str
@return: A list of matched URIs from the currently connected server.
@rtype: list
@raise SnapValueError: One or more of the URI patterns in pattern_list refer to remote servers (absolute).
"""
# Copy the list so we can made modifications.
pattern_list = list(pattern_list)
for (i, pattern) in enumerate(pattern_list):
if pattern == '*':
# Need to map this shortcut to a proper URI path pattern. Also make it
# recursive since this is the old behavior.
pattern_list[i] = "/*"
recursive = True
elif pattern.startswith(connres.server_url):
pattern_list[i] = pattern[len(connres.server_url):]
elif not pattern.startswith('/'):
if pattern.startswith('http://') or pattern.startswith('https://'):
raise CmdSyntaxError("absolute URI '%s' not allowed" % pattern)
else:
raise CmdSyntaxError("invalid URI '%s'." % pattern)
# Get list of resources from server and perform glob matching.
listing = snapi_base.summarize_resources(connres.server_url, cred=connres.get_creds())
server_uri_list = convert_uri_list(listing.keys(), UriStyle.RELATIVE)
matched_uris = set()
for pattern in pattern_list:
matched_uris = matched_uris.union(uri_utils.path_glob_match(pattern,
server_uri_list,
recursive))
return convert_uri_list(matched_uris, uri_style)
class OptionParseError(Exception):
pass
class SAOptionParser(OptionParser):
def __init__(self):
OptionParser.__init__(self, add_help_option=False)
self.examples = None
def error(self, msg):
printerr(msg)
raise OptionParseError()
def text_block_to_lines(self, block):
"""
Convert a string containing multiple lines into a list of lines.
If the string contains multiple lines separated by the \n character, these lines
will be split apart and returned as a list.
Since most strings with multiple lines will be generated using the triple-quote syntax,
they will also likely be aligned with a prefix of whitespace. This function will take care
of stripping the common prefix of whitespace.
Example:
text_block_to_lines('''
This is a multiline
string with alignment whitespace
prefixed to it.
''')
=> ["This is a multiline", "string with alignment whitespace", "prefixed to it."]
@param block: A string containing a block of text with optional line separation characters and
alignment whitespace prefixes.
@type block: str
@return: A list of the split lines with common whitespace prefixes removed.
@rtype: list
"""
block = block.strip('\n')
return textwrap.dedent(block).split('\n')
def format_description(self, formatter):
"""
This method overrides that of the parent Cmd class.
This method was overridden to provide support for some extensions to the help and command
parsing system. This allows a more detailed help to be provided to users.
"""
lines = self.text_block_to_lines(self.description)
description = formatter.format_description('\n'.join(lines))
if self.examples is not None:
examples = self.text_block_to_lines(self.examples)
description += "\nExamples:\n" + "\n".join(examples)
return description
##### common functionality for snapadmin #########################################
class BFSubCmd(Cmd):
""" generic Subcommand class for nested commands """
def __init__(self):
Cmd.__init__(self)
self._create_parser_help_methods()
def command(self, cmdline):
""" dispatcher for subcommands """
if len(cmdline) == 0 :
self.default(cmdline)
else:
args = tokenize(cmdline)
if args[0] == 'help':
# Redirect to standard help code path ('help <cmd> <subcmd>') (ticket #390)
processor.do_help('%s %s' % (self.__class__.__name__, ' '.join(args[1:])))
return
# Cmd.precmd() isn't called by Cmd.onecmd(), so we do it here.
cmdline = self.precmd(cmdline)
(cmd, ignore, ignore) = self.parseline(cmdline)
try:
processor.current_cmd = cmd
except AttributeError:
# Command wasn't found. Let onecmd take care of the error message
processor.current_cmd = ""
self.onecmd(cmdline)
def parse_args(self, cmdline):
try:
parse_func = getattr(self, "parse_" + processor.current_cmd)
except AttributeError:
printerr("Internal error parsing argments.")
raise OptionParseError()
parser = SAOptionParser()
parse_func(parser)
return parser.parse_args(tokenize(cmdline))
def _show_sub_cmd_help(self, parse_func):
parser = SAOptionParser()
parse_func(parser)
parser.print_help()
print
def _create_help_method(self, subcmd, parse_func):
"""
Create a command help method for a parse function.
"""
setattr(self.__class__, 'help_' + subcmd, lambda inst: inst._show_sub_cmd_help(parse_func))
def _create_parser_help_methods(self):
"""
Create help methods for all parse functions on this object.
The created help method will be attached to the class object instead of the instance. This is necessary
due to the way that Cmd does help handling.
"""
for attr in dir(self):
if attr.startswith('parse_') and attr != 'parse_args':
subcmd = attr[6:]
parse_func = getattr(self, attr)
self._create_help_method(subcmd, parse_func)
def print_topics(self, header, cmds, cmdlen, maxcol):
# XXX Prevent the printing of the "help" undocumented command.
if 'help' in cmds:
cmds.remove('help')
return Cmd.print_topics(self, header, cmds, cmdlen, maxcol)
##### component commands ############################################################
class component(BFSubCmd):
""" component commands """
doc_header = 'component help (type help component <command> for details)'
def do_list(self, cmdline):
"""
List components
Usage: component list
Example:
component list
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
comps = snapi_base.list_components(connres.server_url, connres.get_creds())
printok ("The component list command succeeded.")
for comp in comps.keys():
print ' ' + comp
def do_print(self, cmdline):
"""
Print a component
Usage: component print compname
Example:
component print CsvCalc
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
args = tokenize(cmdline)
if len(args) < 1:
printerr("component print command syntax error")
self.do_help('print')
return
comp = args[0]
comps = snapi_base.list_components(connres.server_url, connres.get_creds())
if comp in comps:
print "Component %s:" % comp
printdata(comps[comp], 2)
else:
printerr("Component %s not found on current server." % comp)
##### log commands ############################################################
class log(BFSubCmd):
""" log commands """
doc_header = 'log help (type help log <command> for details)'
def parse_search(self, parser):
parser.usage = """log search [-l limit] [-o offset] pipeline_rid [regex_search_string]
Displays pipeline logs given pipeline runtime ID.
You can find out pipeline runtime ID using the "pipeline list" command.
If an optional regex search string is specified only the log lines
matching the search string will be displayed.
Examples:
-----------
Search for all log messages given pipeline runtime ID:
log search 5418565463186641793300605001041649754243240872
Same as above, but only show messages containing string ERROR:
log search 5418565463186641793300605001041649754243240872 ERROR
Use a regex to search for string ERROR or INFO:
log search 5418565463186641793300605001041649754243240872 "ERROR|INFO"
"""
parser.description = ""
parser.add_option("-l", "--limit",
dest="limit", type="int", default=0,
help="How many log lines to show. Default is 0 (show all).")
parser.add_option("-o", "--offset",
dest="offset", type="int", default=0,
help="Log lines at beginning to skip")
return parser
def do_search(self, cmdline):
if not connres.is_connected():
printerr("Must be connected to a server.")
return
(options, args) = self.parse_args(cmdline)
if len(args) < 1 or len(args) > 2:
printerr("log search command syntax error")
self.do_help('search')
return
pipeline_rid = args[0]
if len(args) == 2:
search_str = args[1]
else:
search_str = None
server_logs = snapi_base.get_server_logs(connres.server_url, cred=connres.get_creds())
fetch_uri = server_logs['pipeline']['all'] + '?' + urllib.urlencode({'rid' : pipeline_rid})
resp = snapi_base.urlopen('GET', url=fetch_uri, headers={'Accept' : 'application/json'})
loglines = simplejson.loads(resp.read())
assert type(loglines) is list
if search_str:
regex = re.compile(search_str)
loglines = [line for line in loglines
if regex.search(line)]
total_log_lines = len(loglines)
start, stop = options.offset, None
if options.limit:
stop = options.offset + options.limit
loglines = loglines[start:stop]
if search_str:
printok("Log search for %s matched %d lines (showing %d):" % (search_str, total_log_lines, len(loglines)))
else:
printok("Log search returned %d lines (showing %d):" % (total_log_lines, len(loglines)))
for logline in loglines:
print ' ' + logline.strip()
# def do_pipeline(self, cmdline):
# TODO: implement an analogue of the 'log print' command in beale's snapadmin
# see ticket #969
##### Pipeline Commands ######################################################
class pipeline(BFSubCmd):
""" pipeline commands """
doc_header = 'pipeline help (type help pipeline <command> for details)'
def do_list(self, cmdline):
"""
List pipelines
Usage: pipeline list
Example:
pipeline list
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
pipelines = connres.get_pipelines()
stat = connres.get_runtime_status()
if pipelines:
printok ("Pipelines found:")
for uri in pipelines:
# Cut off server prefix
uri = uri[len(connres.server_url):]
"""
Print pipeline URI
followed by runtime ID and start times (only for running pipelines)
E.g. output could be:
Success: Pipelines found:
/bench/1x512/p_fw_1x512
Started: Fri Jun 19 13:15:46 2009 Runtime ID: 4984799935442582511647632944730952472817341622
/test/XmlWrite/TestAutoView2View/TestAutoView2View
Note that the first pipeline has running instances, but the second doesn't
have any instances running.
"""
print ' ' + uri
for status in stat:
# If URIs match and pipeline is running print the runtime ID and start time
if status['resource_uri'] == uri:
# Make another request to get runtime status to obtain the start time
pipeline_status = get_status(status['runtime_status_uri'])
print ' %s: %s Runtime ID: %s' % \
(
str(status['state']),
time.ctime(pipeline_status.create_ts),
status['runtime_id']
)
else:
printok("No pipelines.")
def do_print(self, cmdline):
"""
Print a pipeline
Usage: pipeline print pipeline-uri
Example:
pipeline print /resdef/pass/pipelineReadWrite
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
args = tokenize(cmdline)
if len(args) < 1:
printerr("pipeline print command syntax error")
self.do_help('print')
return
rel_uri = args[0]
uri = connres.server_url.rstrip('/') + '/' + rel_uri.lstrip('/')
try:
res = snapi_base.read_resource(uri)[uri]
printok("Found pipeline at %s" % rel_uri)
print(" uri : %s" % uri)
print(" gen_id : %s" % res[keys.GEN_ID])
print(" guid : %s" % res[keys.GUID])
print(" resources : %s" % ','.join(res[keys.RESDEF][keys.PIPELINE_RESOURCES].keys()))
print(" description : %s" % res[keys.RESDEF][keys.DESCRIPTION])
except:
printerr("No pipeline found at %s" % uri)
return
def do_start(self, cmdline):
"""
Start a pipeline
Usage: pipeline start pipeline-uri
Example:
pipeline start /resdef/pass/pipelineReadWrite
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
args = tokenize(cmdline)
if len(args) < 1:
printerr("pipeline start command syntax error")
self.do_help('start')
return
rel_uri = args[0]
uri = connres.server_url.rstrip('/') + '/' + rel_uri.lstrip('/')
handle = exec_resource(uri, credentials = connres.get_creds())
printok("Pipeline started with runtime ID %s" % handle.rid)
def do_stop(self, cmdline):
"""
Stop a pipeline
Usage: pipeline stop runtime-id
Example:
pipeline stop 27600921744567380369450212921456506972228053538089
You can get the runtime ID through the "pipeline list" command.
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
args = tokenize(cmdline)
if len(args) < 1:
printerr("pipeline stop command syntax error")
self.do_help('stop')
return
runtime_id = args[0]
# Given pipeline runtime ID find its runtime status URI,
# since SNAPI needs the runtime status URI to be able to stop the pipeline
stat = connres.get_runtime_status()
runtime_control_uri = None
found = False
for status in stat:
if status['runtime_id'] == runtime_id:
# We found the runtime status URI given the runtime ID
runtime_control_uri = status.get('runtime_control_uri')
found = True
break
if not found:
# Incorrect runtime ID
printerr('Cannot find resource with runtime ID %s' % runtime_id)
return
if runtime_control_uri is None:
printerr('User does not have permission to stop resource with runtime ID %s' % runtime_id)
return
# Send the STOP command to the pipeline
send_stop(runtime_status_uri)
printok("Stop request sent for resource %s" % runtime_id)
class users(BFSubCmd):
""" user related commands """
doc_header = "users help (type help users <command> for details)"
def _read_password_file(self, fhandle):
"""
Read the username/password file.
Returns a dictionary, with username as key and the crypted password
as value.
A file handle of the already opened file is passed in. The file
also needs to be closed by the caller. Any locking is the responsibility
of the caller.
"""
d = {}
fhandle.seek(0)
try:
for (line_number, line) in enumerate(fhandle):
line = line.strip()
if not line.startswith("#") and len(line)>0:
_elems = line.split(":")
user = _elems[0]
password = _elems[1]
d[user] = password
except Exception, e:
raise SnapFormatError("Format error in line %d of password file." % (line_number+1))
return d
def _write_password_file(self, fhandle, udict):
"""
Write a new password file based on the user/password dictionary.
All previous contents of the file will be overwritten.
"""
fhandle.seek(0)
fhandle.truncate(0)
fhandle.write("\n# This file was autogenerated by snapadmin. Do not edit by hand!\n")
fhandle.write("# Last modification: %s\n\n" % time.strftime("%d %b %Y, %H:%M:%S", time.localtime()))
# Sort the list by username
users = udict.keys()
users.sort()
for username in users:
fhandle.write("%s:%s::\n" % (username, udict[username]))
fhandle.flush()
def do_makepasswordfile(self, cmdline):
"""
Create a new password file, only containing the admin user.
Specifying the name of an already existing password file
will overwrite the exiting file. All user information will
be lost.
An admin user will always be created. If the admin password is
not specified on the command line then it will be prompted for.
usage: users makepasswordfile [-f] <password_file> [<admin_password>]
"""
tokens = tokenize(cmdline)
try:
if tokens[0] != "-f":
if len(tokens) < 1 or len(tokens) > 2:
raise IndexError
filename = tokens[0]
try:
password = tokens[1]
except IndexError:
password = getpass.getpass('Password: ')
try:
# Just test if the file exists already
f = open(filename, "r")
f.close()
print "A file '%s' already exists. All its contents will be lost. Proceed?" % filename
if not confirmed():
printok("Creation of new user/password file aborted.")
return
except:
# Looks like the file didn't exist. No problem.
pass
else:
if len(tokens) < 2 or len(tokens) > 3:
raise IndexError
# We will create the new file, no matter if there is one by that name already.
filename = tokens[1]
try:
password = tokens[2]
except IndexError:
password = getpass.getpass('Password: ')
except IndexError:
printerr("users makepasswordfile requires parameters: [-f] <password_file> [<admin_password>]")
return
try:
with open(filename, "w+") as pfile:
file_lock.lock(pfile, file_lock.LOCK_EX)
self._write_password_file(pfile, { "admin" : md5crypt(password, make_salt())})
except IOError, e:
printerr("Cannot open password file '%s' for read/write access. Error: %s" % (filename, e))
return
def do_list(self, cmdline):
"""
Print list of users in the username/password file.
Snapadmin must be connected to a server.
usage: users list
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
tokens = tokenize(cmdline)
if len(tokens) != 0:
printerr("users list does not allow any parameters")
return
if not connres.is_connected():
printerr("Must be connected to a server.")
return
ret = snapi.get_user_list(connres.server_url, connres.get_creds())
for u in ret:
print ' ' + u
printok("ok")
def do_create(self, cmdline):
"""
Create a new user.
usage: users create <username> [<password>]
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
tokens = tokenize(cmdline)
if len(tokens) < 1 or len(tokens) > 2:
printerr("users create requires parameters: <username> [<password>]")
return
username = tokens[0]
try:
password = tokens[1]
except IndexError:
password = getpass.getpass('Password: ')
user = snapi.create_user_entry(connres.server_url, username=username, password=password, credentials=connres.get_creds())
user.save()
printok("User '%s' has been created." % username)
def do_setpassword(self, cmdline):
"""
Modify the password of an existing user.
usage: users setpassword <username> [<password>]
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
tokens = tokenize(cmdline)
if len(tokens) < 1 or len(tokens) > 2:
printerr("users setpassword requires parameters: <username> [<password>]")
return
username = tokens[0]
try:
password = tokens[1]
except IndexError:
password = getpass.getpass('Password: ')
user = snapi.get_user_entry(connres.server_url, username=username, credentials=connres.get_creds())
user.set_password(password)
user.save()
printok("Password for user '%s' has been set." % username)
def do_delete(self, cmdline):
"""
Delete an existing user from the username/password file.
usage: users delete [-f] <username>
The -f option specifies 'force'. This means that the user
will be deleted, without prompt.
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
usage_err = "users delete requires parameters: [-f] <password_file> <username>"
tokens = tokenize(cmdline)
if len(tokens) < 1 or len(tokens) > 2:
printerr(usage_err)
return
if tokens[0] == "-f":
if len(tokens) != 2:
printerr(usage_err)
return
tokens.pop(0)
else:
if len(tokens) != 1:
printerr(usage_err)
return
print "Deletion of users cannot be undone. Do you want to continue?"
if not confirmed():
printok("Deletion of user aborted.")
return
username = tokens[0]
user = snapi.get_user_entry(connres.server_url, username=username, credentials=connres.get_creds())
user.delete()
printok("User '%s' has been deleted." % username)
##### Repository Commands ######################################################
class repository(BFSubCmd):
""" repository commands """
doc_header = 'repository help (type help repository <command> for details)'
def add_repo_options(self, parser):
parser.add_option("-c", "--config",
dest="config", type="string", metavar="<config-file>",
help="Read repository location from config file.")
parser.add_option("-t", "--type",
dest="type", type="choice", choices=["mysql", "sqlite"],
help="Type of repository database to create.")
parser.add_option("-H", "--host",
dest="host", type="string", default="localhost",
help="MySQL server hostname or IP.")
parser.add_option("-P", "--port",
dest="port", type="int",
help="MySQL server hostname or IP.")
parser.add_option("-u", "--user",
dest="user", type="string",
help="MySQL username.")
parser.add_option("-p", "--password",
dest="passwd", type="string",
help="MySQL password.")
def validate_repo_options(self, cmd, options, args):
if not options.type and not options.config:
printerr("%s requires a repository type (-t) or config file (-c)." % cmd)
return None
elif options.type == "sqlite":
if len(args) != 1:
printerr("SQLite %s requires one data file." % cmd)
return None
else:
repo_config = {'type': 'sqlite', 'path': args[0]}
elif options.type == "mysql":
if not self.validate_mysql_options(cmd, options, args):
return None
else:
repo_config = {'type': 'mysql',
'host': options.host,
'db': args[0],
'user': options.user}
if options.port is not None:
repo_config['port'] = options.port
if options.passwd is not None:
repo_config['password'] = options.passwd
elif options.type is None and options.config:
repo_config = self.parse_config_file(cmd, options.config)
if repo_config is None:
return None
else:
printerr("Invalid repository type.")
return None
return repo_config
def validate_mysql_options(self, cmd, options, args):
if not options.user:
printerr(cmd + " failed: username required (-u).")
return False
if len(args) == 0:
printerr(cmd + " failed: MySQL database name required.")
return False
if len(args) > 1:
printerr(cmd + " failed: only one MySQL database name allowed.")
return False
if options.passwd is None:
options.passwd = getpass.getpass('Enter MySQL password: ')
return True
def parse_config_file(self, cmd, filename):
try:
# Since snapadmin is designed for reloading config files
# tell snap config to not enforce the singleton.
# This simply means that it won't throw an exception
# if config object gets created more than once.
snap_config.enforce_singleton = False
# Make a snap config object by reading the config file.
config = SnapConfig(filename)
except Exception, e:
printexc(e)
return None
try:
return config.get_section('repository')
except Exception, e:
printerr(cmd + " failed: no repository section in config file.")
return None
def parse_repo_options(self, cmd, cmdline):
(options, args) = self.parse_args(cmdline)
repo_config = self.validate_repo_options(cmd, options, args)
return (options, repo_config)
def parse_create(self, parser):
parser.usage = "repository create -t <db-type> [options] ..."
parser.description = """
Create a repository of the specified database type. The database type can be either
sqlite (preferred) or mysql. The possible options vary depending on the database
type.
"""
parser.examples = """
repository create -t sqlite repository.db
repository create -t mysql -H mysql.domain.com -u snaplogic repository
"""
self.add_repo_options(parser)
return parser
def do_create(self, cmdline):
"""
Create a new repository.
usage: repository create -t mysql [-H <host>] [-P <port>] -u <username> [-p <password>] <database>
repository create -t sqlite <filename>
Example:
repository create -t mysql -u root -p root snapdb
repository create -t sqlite /tmp/snap.db
"""
(options, repo_config) = self.parse_repo_options('repository create', cmdline)
if repo_config is None:
return
elif repo_config['type'] == 'sqlite':
self.createSQLite(repo_config)
elif repo_config['type'] == 'mysql':
self.createMySQL(repo_config)
def createMySQL(self, repo_config):
try:
from snaplogic.server.repository.mysql import MySQL
MySQL.create(repo_config['host'],
repo_config.get('port', None),
repo_config['db'],
repo_config['user'],
repo_config['password'])
printok("MySQL repository created.")
print "Sample repository section for this database:"
print "[repository]"
print "type = mysql"
print "host =", repo_config['host']
if 'port' in repo_config:
print "port =", repo_config['port']
print "db =", repo_config['db']
print "user =", repo_config['user']
if 'password' in repo_config:
print "password =", repo_config['password']
except SnapException, e:
printerr("repository create failed: ")
printexc(e)
return
def createSQLite(self, repo_config):
try:
from snaplogic.server.repository.sqlite import SQLite
SQLite.create(repo_config['path'])
except SnapException, e:
printerr("Error creating repository database file '%s'" % (repo_config['path']))
printexc(e)
return
printok("Repository database file '%s' created." % repo_config['path'])
print "Sample repository section for this database:"
print "[repository]"
print "type = sqlite"
print "path = " + repo_config['path']
def destroy_sqlite(self, repo_config):
from snaplogic.server.repository.sqlite import SQLite
SQLite.destroy(repo_config['path'])
def destroy_mysql(self, repo_config):
from snaplogic.server.repository.mysql import MySQL
MySQL.destroy(repo_config['host'],
repo_config.get('port', None),
repo_config['db'],
repo_config['user'],
repo_config.get('password', None))
def parse_destroy(self, parser):
parser.usage = "repository destroy {-t <db-type> | -c <config-file>} [options] ..."
parser.description = """
Destroy a repository of the specified database type. The database type can be either
sqlite or mysql. The possible options vary depending on the database
type.
If the config file option (-c) is used instead, the repository location and
credentials will be determined from the config file given.
"""
parser.examples = """
repository destroy -t sqlite repository.db
repository destroy -t mysql -H mysql.domain.com -u snaplogic repository
repository destroy -c /home/snaplogic/config/snapserver.conf
"""
self.add_repo_options(parser)
parser.add_option("-f", "--force",
dest="force", action="store_true", default=False,
help="Force destroy without prompting")
return parser
def do_destroy(self, cmdline):
(options, repo_config) = self.parse_repo_options('repository destroy', cmdline)
if repo_config is None:
return
if not options.force:
print "Destroying this repository will remove all resource and metadata"
print "stored within the database. This operation is not reversible."
if not confirmed():
printok("Repository destroy aborted.")
return
repo_type = repo_config['type']
if repo_type == 'sqlite':
self.destroy_sqlite(repo_config)
elif repo_type == 'mysql':
self.destroy_mysql(repo_config)
printok("Repository destroyed.")
print "The database or file (sqlite) has been left and may be reinitialized"
print "with the 'repository create' command, or you may delete it."
def parse_upgrade(self, parser):
parser.usage = "repository upgrade {-t <db-type> | -c <config-file>} [options] ..."
parser.description = """
Upgrades the structure of the repository database. This command is used by the
SnapLogic installer. In general it should not be used manually unless
you are sure of what you are doing.
"""
parser.examples = """
repository upgrade -t sqlite repository.db
repository upgrade -t mysql -H mysql.domain.com -u snaplogic repository
repository upgrade -c /home/snaplogic/config/snapserver.conf
"""
self.add_repo_options(parser)
return parser
def do_upgrade(self, cmdline):
"""
Upgrade a repository's internal structure.
usage: repository upgrade -t mysql [-H <host>] [-P <port>] -u <username> [-p <password>] <database>
repository upgrade -t sqlite <filename>
repository upgrade -c <filename>
Example:
repository upgrade -t sqlite /tmp/snap.db
repository upgrade -c /opt/Snap/SnapServer.conf
"""
(options, repo_config) = self.parse_repo_options('repository upgrade', cmdline)
if repo_config is None:
return
store = repo_module.connect_store(repo_config, False)
old_version = store.read_version()
print "Beginning repository upgrade from format %d..." % old_version
store.upgrade_store()
printok("Upgrade complete.")
def default(self, cmdline):
printerr("repository command syntax error")
self.do_help('')
##### Resource commands ########################################################
class resource(BFSubCmd):
""" resource commands """
doc_header = 'resource help (type help resource <command> for details)'
def _make_relative(self, uri):
"""
Convert an absolute URI to relative.
@param uri: Absolute URI.
@type uri: string
@return: Reltaive URI.
@rtype: string
"""
if uri[0] == '/':
return uri
else:
return snap_http_lib.parse_uri(uri).path
def _convert_absolute_uris(self, resdef_dict, local_only=True):
resdef = snapi_resdef.create_from_dict(resdef_dict)
if isinstance(resdef, snapi_resdef.PipelineResDef):
resdef.make_resources_relative(connres.server_url if local_only else None)
return resdef.dict
def _contains_absolute_references(self, resdef_dict):
resdef = snapi_resdef.create_from_dict(resdef_dict)
if isinstance(resdef, snapi_resdef.PipelineResDef):
for res_name in resdef.list_resource_names():
res_uri = resdef.get_resource_uri(res_name)
if not res_uri.startswith('/'):
return True
return False
def _glob_match_package_contents(self, package, pattern_list, recursive=False):
package_uri_list = []
for (uri, resdef) in package:
package_uri_list.append(uri)
matched_uris = set()
for pattern in pattern_list:
matched_uris = matched_uris.union(uri_utils.path_glob_match(pattern,
package_uri_list,
recursive))
return matched_uris
def _uri_list_difference_by_relative_path(self, uri_list1, uri_list2):
set1 = set([self._make_relative(uri) for uri in uri_list1])
return list(set1.difference([self._make_relative(uri) for uri in uri_list2]))
def do_print(self, cmdline):
"""
Print a resource
usage: resource print <resource_uri>|all
Example:
resource print /resdef/crm/customer
resource print all
"""
if not connres.is_connected():
printerr("Must be connected to a server.")
return
args = tokenize(cmdline)
if len(args) < 1:
printerr("resource print requires a resource uri, or all")
self.do_help('print')
return
if args[0].lower() == 'all' or args[0] == '*':
listing = snapi_base.list_resources(connres.server_url, cred=connres.get_creds())
resources = snapi_base.read_resources(listing.keys(), cred=connres.get_creds())
if resources is None:
resources = {keys.SUCCESS: None, keys.ERROR: None} # slight hack...
else:
urls = []
for a in args:
if a[0] == '/':
urls.append(connres.server_url + a)
else:
urls.append(a)
resources = snapi_base.read_resources(urls, cred=connres.get_creds())
if resources[keys.ERROR]:
for (uri, error) in resources[keys.ERROR].iteritems():
printerr("Error reading resource '%s': %s" % (uri, error))
return
elif not resources[keys.SUCCESS]:
print "No resources."
return
for (uri, info) in resources[keys.SUCCESS].iteritems():
print uri + ":"
print "GUID: ", info[keys.GUID]
print "GenID: ", info[keys.GEN_ID]
print "ResDef:"
pprint.pprint(info[keys.RESDEF])
print
def parse_list(self, parser):
parser.usage = "resource list [options] [<uri-pattern> ...]"
parser.description = """List resources in the repository."""
parser.examples = """
List all resources:
resource list
List all resources in the /test folder (non-recursive):
resource list /test/*
List all resources in /test1 and all subfolders (resursive):
resource list -r /test
List all resources in /test1, /test2, and /test3 folders (non-recursive):
resource list /test[1-3]/*
List all resources whose component name contains 'CsvRead':
resource list -m CsvRead
"""
parser.add_option("-a", "--absolute",
dest="absolute", action="store_true", default=False,
help="Print absolute URIs")
parser.add_option("-c", "--component",
dest="component", metavar="<search>",
help="Show only resources whose component name contains <search>")
parser.add_option("-r", "--recursive",
dest="recursive", action="store_true", default=False,
help="Descend recursively into matched subfolders")
return parser
def do_list(self, cmdline):
if not connres.is_connected():
printerr("Must be connected to a server.")
return
(options, uri_list) = self.parse_args(cmdline)
uri_style = UriStyle.ABSOLUTE if options.absolute else UriStyle.RELATIVE
details = None if options.component is None else [keys.COMPONENT_NAME]
if uri_list:
# Get list of matched URIs from argument patterns. We get the URIs as absolute regardless
# of optoins specified to make the output consistent with summarize_resources.
matched_uris = match_server_uris(uri_list, options.recursive, UriStyle.ABSOLUTE)
if matched_uris and options.component is not None:
summary = snapi_base.summarize_resources(connres.server_url,
convert_uri_list(matched_uris, UriStyle.RELATIVE),
details,
cred=connres.get_creds())
else:
summary = snapi_base.summarize_resources(connres.server_url,
details=details,
cred=connres.get_creds())
matched_uris = summary.keys()
# If the user specified a component string match, filter any resource whose component name
# does not contain the specified string.
if options.component is not None:
def component_filter(uri):
cname = summary[uri][keys.SUMMARY][keys.COMPONENT_NAME]
return cname.find(options.component) != -1
matched_uris = filter(component_filter, matched_uris)
if matched_uris:
# The values in matched_uris will be what is printed to the user. Convert them to whatever form the
# user selected.
matched_uris = convert_uri_list(matched_uris, uri_style)
matched_uris.sort()
for uri in matched_uris:
print uri
elif uri_list:
printok("No matching resources found.")
else:
printok("No resources found.")
def parse_delete(self, parser):
parser.usage = "resource delete [options] {<uri-patterns> | *}"
parser.description = "Delete resources from the repository."
parser.examples = """
Delete a single resource:
resource delete /resdef/crm/customer
Delete multiple resources:
resource delete /resdef/crm/pipe /resdef/crm/writer
Delete all resources located in /resdef/crm folder:
resource delete /resdef/crm/*
Delete all resources located in /test and any subfolders:
resource delete -r /test
Delete all resources without prompting for confirmation:
resource delete -f *
"""
parser.add_option("-f", "--force",
dest="force", action="store_true", default=False,
help="Force deletion resources")
parser.add_option("-r", "--recursive",
dest="recursive", action="store_true", default=False,
help="Descend recursively into matched subfolders")
return parser
def do_delete(self, cmdline):
if not connres.is_connected():
printerr("Must be connected to a server.")
return
(options, uri_list) = self.parse_args(cmdline)
if len(uri_list) < 1:
printerr("resource delete missing URI list.")
return
# Get list of matched URIs from argument patterns and convert to absolute addresses.
matched_uris = match_server_uris(uri_list, options.recursive)
matched_uris = [connres.server_url + uri for uri in matched_uris]
count = 0
for uri in matched_uris:
try:
if not options.force:
print "Deleting %s..." % uri
if not confirmed():
continue
snapi.delete_resource(uri, connres.get_creds())
print "Deleted %s" % uri
count += 1
except SnapiHttpException, e:
if e.status == HttpRequest.NOT_FOUND:
# Someone deleted this resource before we could.
pass
elif e.status == HttpRequest.CONFLICT:
print "Warning: resource %s changed while deleting." % uri
else:
raise
printok("Deleted %d resources." % count)
def parse_import(self, parser):
parser.usage = "resource import [options] {<uri-patterns> | *}"
parser.description = """
Import resources into the server repository from a dump file created with
'resource export'.
"""
parser.examples = """
Import all resources in snaplogic.dmp located in the current directory:
resource import *
Import all resources from /tmp/res.dmp:
resource import -i /tmp/res.dmp *
Import all resources in folder /test (non-recursive):
resource import /test/*
Import all resources in folder /test and all sub-folder (recursive):
resource import -r /test
Import all resources in /test and store in /prod folder on server:
resource import -s /test=/prod /test/*
"""
parser.add_option("-f", "--force",
dest="force", action="store_true", default=False,
help="Force overwrite of existing resources")
parser.add_option("-i", "--input",
dest="file", metavar="<filename>", default="snaplogic.dmp",
help="Input file [default %default]")
parser.add_option("-r", "--recursive",
dest="recursive", action="store_true", default=False,
help="Descend recursively into matched subfolders")
parser.add_option("-s", "--substitute",
dest="substitution", metavar="<old-text>=<new-text>",
help="Substitute <orig-text> with <new-text> when naming resources and pipelines")
parser.add_option("-S", "--Substitute",
dest="Substitution", metavar="<old-text>=<new-text>",
help="Substitute <orig-text> with <new-text> when naming resources and pipelines,\n" +
"but keep references to resources in newly imported pipelines unchanged.")
parser.add_option("-A", "--ignore-absolute",
dest="ignore_absolute", action="store_true", default=False,
help="Ignore absolute resource URIs in pipelines")
parser.add_option("-C", "--ignore-missing-component",
dest="ignore_missing_component", action="store_true", default=False,
help="Ignore missing component error")
parser.add_option("-R", "--relative",
dest="relative", action="store_true", default=False,
help="Convert absolute URIs to relative")
parser.add_option("-U", "--no-upgrade",
dest="upgrade", action="store_false", default=True,
help="Import resources without upgrading them")
return parser
def do_import(self, cmdline):
(options, uri_list) = self.parse_args(cmdline)
substitution = None
if not uri_list:
printerr("resource import missing list of import URIs.")
return
elif options.substitution or options.Substitution:
substitution = options.substitution if options.substitution else options.Substitution
superficial = substitution == options.Substitution
(subst_orig, subst_new) = substitution.split('=', 1)
if subst_orig == '':
printerr("resource import failed: substitution original text must not be empty.")
return
elif subst_orig[0] == '/' and (not subst_new or subst_new[0] != '/'):
printerr("resource import failed: substitution new text does not begin with '/'.")
return
for (i, uri) in enumerate(uri_list):
if uri == '*':
# Need to map this shortcut to a proper URI path pattern. Also make it
# recursive since this is the old behavior.
uri_list[i] = "/*"
options.recursive = True
elif not uri.startswith('/'):
printerr("resource import failed: absolute URI '%s' not allowed." % uri)
return
if not connres.is_connected():
printerr("Must be connected to a server.")
return
try:
component_list = snapi.list_components(connres.server_url, credentials=connres.get_creds()).keys()
except Exception, e:
printerr("resource import failed: error retrieving component list: ")
printexc(e)
return
# Add the pseudocomponent Pipeline into the list
component_list.append(keys.PIPELINE_COMPONENT_NAME)
imported_uri_list = []
try:
with open(options.file, "r") as import_file:
# Do an initial run through the file to get the entire URI path structure for globbing
# and error checking.
package = PackageReader(import_file)
matched_uris = self._glob_match_package_contents(package, uri_list, options.recursive)
# Reset the file back to the beginning and iterate through the package again, importing the
# URIs that were matched.
import_file.seek(0)
package = PackageReader(import_file)
for (uri, resdef) in package:
if uri not in matched_uris:
continue
# Check that component name of resdef exists unless option to ignore was given.
if not options.ignore_missing_component and resdef[keys.COMPONENT_NAME] not in component_list:
cname = resdef[keys.COMPONENT_NAME]
printerr("Ignoring resource '%s' with missing component name '%s'." % (uri, cname))
continue
# Perform URI transformations requested in options.
new_uri = uri
if substitution:
new_uri = uri.replace(subst_orig, subst_new)
# If this is a pipeline that references other resources
# make sure the references have been substituted as well.
# However if they use "superficial" mode skip this step.
# In superficial mode we don't touch resource references inside pipeline.
if resdef[keys.COMPONENT_NAME] == keys.PIPELINE_COMPONENT_NAME and not superficial:
pipeline_resources = resdef[keys.PIPELINE_RESOURCES]
# Iterate through pipeline resources
for (resource_name, resource_resdef) in pipeline_resources.items():
resource_uri = resource_resdef[keys.URI]
# Have we renamed this resource? If yes updated the reference in the pipeline.
if resource_uri in matched_uris:
resource_resdef[keys.URI] = resource_uri.replace(subst_orig, subst_new)
if options.relative:
resdef = self._convert_absolute_uris(resdef, False)
elif not options.ignore_absolute and self._contains_absolute_references(resdef):
print "Resource '%s' contains absolute URIs to pipeline resources. Make relative?" % uri
if confirmed():
resdef = self._convert_absolute_uris(resdef, False)
try:
res_url = connres.server_url + new_uri
snapi_base.write_resource(resdef, uri=res_url, cred=connres.get_creds())
except SnapiHttpException, e:
if e.status != HttpRequest.CONFLICT:
raise
elif not options.force:
print "Resource '%s' already exists. Continuing will overwrite this resource." \
% new_uri
if not confirmed():
continue
listing = snapi_base.list_resources(connres.server_url, [new_uri], cred=connres.get_creds())
res_info = listing.values()[0]
if res_info is not None:
try:
snapi_base.write_resource(resdef,
res_info[keys.GUID],
res_info[keys.GEN_ID],
res_url,
cred=connres.get_creds())
except SnapiException, e:
printerr("resource import failed:")
printexc(e)
return
print "Imported %s as %s" % (uri, new_uri)
imported_uri_list.append(new_uri)
except IOError, e:
printerr("resource import failed to read file: ")
printexc(e)
return
print "Imported %d resources from file '%s'." % (len(imported_uri_list), options.file)
if options.upgrade and imported_uri_list:
result = snapi.upgrade_resources(connres.server_url, imported_uri_list, connres.get_creds())
_print_upgrade_result(result)
def parse_export(self, parser):
parser.usage = "resource export [options] {<uri-patterns> | *}"
parser.add_option("-d", "--dependencies",
action="store_true", dest="dependencies", default=False,
help="Export resource dependencies with resources")
parser.add_option("-o", "--output",
dest="file", metavar="<filename>", default="snaplogic.dmp",
help="Output file [default: %default]")
parser.add_option("-r", "--recursive",
dest="recursive", action="store_true", default=False,
help="Descend recursively into matched subfolders")
parser.add_option("-R", "--relative",
dest="relative", action="store_true", default=False,
help="Convert local absolute URIs to relative")
return parser
def do_export(self, cmdline):
(options, uri_list) = self.parse_args(cmdline)
if not uri_list:
printerr("resource export missing list of export URIs.")
return
if not connres.is_connected():
printerr("Must be connected to a server.")
return
# Get list of matched URIs from argument patterns and convert to absolute addresses.
matched_uris = match_server_uris(uri_list, options.recursive)
matched_uris = [connres.server_url + uri for uri in matched_uris]
try:
read_ret = snapi_base.read_resources(matched_uris, cred=connres.get_creds())
except SnapiException, e:
printerr("resource export failed to read resources:")
printexc(e)
return
if read_ret[keys.ERROR]:
for (uri, error) in read_ret[keys.ERROR].iteritems():
if error is None:
error = "resource does not exist"
printerr("resource export failed to read resource '%s': %s." % (uri, error))
return
elif len(read_ret[keys.SUCCESS]) == 0:
printerr("resource export failed: no resources matched given pattern(s).")
return
resources = read_ret[keys.SUCCESS]
if options.dependencies:
try:
dependencies = set()
for uri in resources:
# Get only the local dependencies.
dependencies = dependencies.union(snapi_base.get_resource_dependencies(uri, True))
# Remove the resources already contained in the user supplied list.
dependencies = self._uri_list_difference_by_relative_path(dependencies, resources)
dependencies = [connres.server_url + uri for uri in dependencies]
read_ret = snapi_base.read_resources(list(dependencies))
except SnapiException, e:
printerr("resource export failed: unable to read dependencies:")
printexc(e)
return
if read_ret[keys.ERROR]:
for (uri, error) in read_ret[keys.ERROR].iteritems():
if error is None:
error = "resource does not exist"
printerr("resource export failed to read resource '%s': %s." % (uri, error))
return
elif len(read_ret[keys.SUCCESS]) != len(dependencies):
printerr("resource export failed to read all dependencies.")
return
resources.update(read_ret[keys.SUCCESS])
dependency_count = len(read_ret[keys.SUCCESS])
else:
dependency_count = 0
try:
with open(options.file, 'w') as export_file:
package = PackageWriter(export_file)
for (uri, info) in resources.iteritems():
if options.relative:
info[keys.RESDEF] = self._convert_absolute_uris(info[keys.RESDEF])
uri = self._make_relative(uri)
package.write_resource(uri, info[keys.RESDEF])
print "Exported " + uri
package.end()
except IOError, e:
printerr("resource export unable to write file:")
printexc(e)
return
if options.dependencies:
print "Exported %d resources and %d dependencies to file '%s'." % (len(resources) - dependency_count,
dependency_count,
options.file)
else:
print "Exported %d resources to file '%s'." % (len(resources), options.file)
def parse_upgrade(self, parser):
parser.usage = "resource upgrade [options] {<uri-patterns> | *}"
parser.description = "Upgrade resources from old component versions to the latest."
parser.add_option("-r", "--recursive",
dest="recursive", action="store_true", default=False,
help="Descend recursively into matched subfolders")
return parser
def do_upgrade(self, cmdline):
(options, uri_list) = self.parse_args(cmdline)
if not uri_list:
printerr("resource upgrade missing list of resource URIs.")
return
if not connres.is_connected():
printerr("Must be connected to a server.")
return
# Get list of matched URIs from argument patterns and convert to absolute addresses.
if '*' not in args:
matched_uris = match_server_uris(uri_list, options.recursive)
else:
matched_uris = None
try:
result = snapi.upgrade_resources(connres.server_url, matched_uris, connres.get_creds())
except SnapiException, e:
printexc(e)
return
_print_upgrade_result(result, True)
def default(self, cmdline):
printerr("resource command syntax error")
##### connect commands #########################################################
class connect(BFSubCmd):
""" connect commands """
doc_header = 'connect help (type help connect <command> for details)'
def disconnect(self):
"""
Disconnect from the currently connected SnapLogic server.
usage: disconnect
"""
global connres
if connres.server_url is None:
printok("Not connected.")
else:
connres.disconnect()
printok("Connection closed.")
def do_server(self, cmdline):
"""
Connect to a SnapLogic server.
usage: connect server <server_url>
Example:
connect server http://myserverhost:8088
"""
args = tokenize(cmdline)
if len(args) < 1:
printerr("connect server requires a server URL.")
return
url = args[0]
connres.connect(url)
def do_switch(self, cmdline):
"""
Switch to use a specified connection
usage: connect switch <connection_id>
Example:
connect switch 2
"""
args = tokenize(cmdline)
if len(args) < 1:
printerr("connect switch requires a connection id.")
return
connres.switch(int(args[0]) - 1)
printok("Connection switched (%s)." % connres.server_url)
def do_list(self, cmdline):
"""
List server connections.
usage: connect list
"""
for (i, info) in enumerate(connres.connections):
print "%d: %s" % (i + 1, info['uri'])
def default(self, cmdline):
print '*** connect syntax error'
self.do_help('')
##### credential commands ############################################################
class credential(BFSubCmd):
""" credential commands """
doc_header = 'credential help (type help credential <command> for details)'
def do_set(self, cmdline):
"""
Set the user credential
usage: credential set <default|current|<connection_id>> <user> [<password>]
Example:
credential set default steve
credential set current steve
credential set 2 steve
"""
args = tokenize(cmdline)
if len(args) < 2:
printerr("credential set command syntax error")
self.do_help('set')
return
if len(args) > 2:
passwd = args[2]
else:
passwd = getpass.getpass('Password: ')
user = args[1]
if args[0] == 'current':
if not connres.is_connected():
printerr("Currently not connected, no credential set.")
print " Use 'default' option if the credential is intended for global use."
return
connres.set_creds(user, passwd)
elif args[0] == 'default':
connres.set_default_creds(args[1], passwd)
else:
idx = int(args[0]) - 1
try:
connres.set_creds(user, passwd, idx)
except IndexError:
printerr("Connection id out of range")
raise
printok("credential is set")
def do_reset(self, cmdline):
"""
Reset the user credential
usage: credential reset [default|current|<connection_id>]
Example:
credential reset current
credential reset 2
credential reset default
credential reset
"""
args = tokenize(cmdline)
if len(args) == 0 or args[0] == 'current':
connres.reset_creds()
elif args[0] == 'default':
connres.reset_default_creds()
else:
idx = int(args[0]) - 1
try:
connres.reset_creds(idx)
except IndexError:
printerr("Connection id out of range")
except Exception, e:
printerr("credential set failed")
printexc(e)
else:
printok("credential is reset")
##### verbose command ############################################################
class verbose(BFSubCmd):
""" verbose command """
doc_header = 'verbose help (type help verbose <command> for details)'
def do_on(self, cmdline):
"""
Verbose mode.
Print additional error information.
Usage: verbose on
"""
global verbose_mode
verbose_mode = True
print "Verbose mode is ON"
def do_off(self, cmdline):
"""
Turn off verbose mode.
Succinct error reporting.
Usage: verbose off
"""
global verbose_mode
verbose_mode = False
print "Verbose mode is OFF"
def do_lasterr(self, cmdline):
"""
Print additional information about the last error, if available.
Usage: verbose lasterr
"""
global verbose_lasterr
if verbose_lasterr:
print "Additional error information"
print "=" * 80
print verbose_lasterr
else:
print "No error occurred."
def default(self, cmdline):
global verbose_mode
print 'Verbose mode is currently %s' % ('ON' if verbose_mode else 'OFF')
self.do_help('')
###### main command loop #######################################################
class snapadmin(Cmd):
""" SnapLogic Administration Utility. """
doc_header = 'snapadmin help (type help <command> for details)'
prompt = 'snapadmin > '
def __init__(self, *a, **kw):
Cmd.__init__(self, *a, **kw)
""" initialize subcommand classes """
self.connect = connect()
self.pipeline = pipeline()
self.resource = resource()
self.users = users()
self.repository = repository()
self.component = component()
self.log = log()
self.credential = credential()
self.verbose = verbose()
def emptyline(self):
# disable command repeat
return
#### Commands
def do_source(self, cmdline):
args = tokenize(cmdline)
if len(args) < 1:
printerr("source requires a file name.")
return
try:
batch_commands(args[0])
except Exception, e:
print "source command failed - %s" % str(e)
printok("source command is executed")
def do_connect(self, cmdline):
self.connect.command(cmdline)
def help_connect(self):
self.connect.do_help(None)
def do_disconnect(self, cmdline):
self.connect.disconnect()
def help_disconnect(self):
print """
Disconnect from a repository.
"""
def do_pipeline(self,cmdline):
self.pipeline.command(cmdline)
def help_pipeline(self):
self.pipeline.do_help(None)
def do_resource(self,cmdline):
self.resource.command(cmdline)
def help_resource(self):
self.resource.do_help(None)
def do_users(self, cmdline):
self.users.command(cmdline)
def help_users(self):
self.users.do_help(None)
def do_repository(self, cmdline):
self.repository.command(cmdline)
def help_repository(self):
self.repository.do_help(None)
def do_component(self, cmdline):
self.component.command(cmdline)
def help_component(self):
self.component.do_help(None)
def do_log(self, cmdline):
self.log.command(cmdline)
def help_log(self):
self.log.do_help(None)
def do_credential(self,cmdline):
self.credential.command(cmdline)
def help_credential(self):
self.credential.do_help(None)
def help_source(self):
print """
Execute a sequence of snapadmin commands from a script.
usage: source <path>
example:
source /home/joey/my_commands.txt
"""
def do_shell(self,cmdline):
ret = os.system(cmdline)
if ret != 0:
printerr("shell command returned exit status %d " % ret)
return False
def help_shell(self):
print """
Execute an OS shell command.
"""
def do_verbose(self, cmdline):
self.verbose.command(cmdline)
def help_verbose(self):
"""
Verbose command help.
Must have this method so verbose is included in the list of "documented" commands.
Actual help is printed by introspecting the doc of the "verbose" class.
"""
pass
def do_bye(self, cmdline):
print 'bye'
return True
def do_exit(self, cmdline):
print 'bye'
return True
def help_exit(self):
print """
exit the snapadmin utility
"""
def help_bye(self):
print """
exit the snapadmin utility
"""
def do_help(self, cmdline):
# Intercept help command and support 'help cmd sub-cmd' syntax.
if not cmdline:
# Fall back to parent class
Cmd.do_help(self, cmdline)
return
# If the first word is a top-level command, look to see if it is a subcommand class. If so,
# allow that class to handle the help. This is to support the 'help cmd sub-cmd' syntax.
(cls_name, arg, ignore) = self.parseline(cmdline)
try:
cls_inst = getattr(self, cls_name)
cls_inst.do_help(arg)
except AttributeError, e:
# The look for a class failed. Allow Cmd to handle help in its default manner.
Cmd.do_help(self, cmdline)
def default(self,cmdline):
if cmdline[0:1] == '#':
print cmdline
return
elif cmdline == 'EOF':
return self.do_bye(cmdline)
else:
Cmd.default(self, cmdline)
def onecmd(self, cmdline):
cmdline = cmdline.replace('\\', '\\\\')
try:
return Cmd.onecmd(self, cmdline)
except OptionParseError:
# Already handled by SAOptionParser
pass
except CmdSyntaxError, e:
printerr("syntax error: " + e.message)
except Exception, e:
printexc(e)
################################################################################
if __name__ == '__main__':
processor = snapadmin()
parser = OptionParser()
parser.add_option("-c", "--command",
dest="batch_file", metavar="<filename>",
help="run a batch of commands from file <filename>")
parser.add_option("-q", "--quiet",
action="store_true", dest="quiet_mode", default=False,
help="print less output")
parser.add_option("-e", "--execute",
dest="execute", metavar="<command>",
help="run <command> as if typed into snapadmin")
parser.add_option("-H", "--history-file",
dest="history_file", metavar="<filename>",
help="load and save command history in file <filename>")
(options, args) = parser.parse_args()
if options.batch_file is not None:
if not os.path.isfile(options.batch_file):
printerr("File '%s' not found." % options.batch_file)
sys.exit(1)
try:
batch_commands(options.batch_file, quiet_mode = options.quiet_mode)
except Exception, e:
print "batch command failed - %s" % str(e)
sys.exit(1)
elif options.execute is not None:
processor.onecmd(options.execute)
else:
if options.history_file is not None and os.path.isfile(options.history_file):
readline.read_history_file(options.history_file)
processor.intro = 'Welcome to snapadmin. \nA help command is available.'
done = False
while not done:
try:
processor.cmdloop()
done = True
except KeyboardInterrupt:
# Ignore Ctrl-C. Also disable banner so it doesn't redisplay when cmdloop starts again.
processor.intro = None
print
if options.history_file is not None:
readline.write_history_file(options.history_file)
|