# $SnapHashLicense:
#
# SnapLogic - Open source data services
#
# Copyright (C) 2008 - 2009, SnapLogic, Inc. All rights reserved.
#
# See http://www.snaplogic.org for more information about
# the SnapLogic project.
#
# This program is free software, distributed under the terms of
# the GNU General Public License Version 2. See the LEGAL file
# at the top of the source tree.
#
# "SnapLogic" is a trademark of SnapLogic, Inc.
#
#
# $
# $Id: __init__.py 9868 2009-11-21 17:50:45Z dhiraj $
""" snaplogic.common Package """
import os
from string import Template
def sqlite_iter(cursor):
"""
Safely generate results of sqlite query
Under certain conditions and system configurations, pysqlite can
suffer from a bug when iterating through a result set using the
"for row in cursor:" idiom. A row will be dropped from the result
set some small fraction of the time.
This function is a generator that produces the results in an
alternative, more robust way. Instead of using "for row in
cursor:", use "for row in sqlite_iter(cursor):".
@param cursor : cursor
@type cursor : pysqlite2.dbapi2.Cursor
@return : 0 or more rows (result set)
@rtype : generated list
"""
BUFR = 100 # how many rows to buffer
while 1:
rows = cursor.fetchmany(BUFR)
if not rows:
break
for row in rows:
yield row
def expand_variables(string_with_vars):
"""
Expand variables in string using OS environment.
Currently only expands SNAP_HOME (and if it's not defined uses dot).
E.g. given this input:
"$SNAP_HOME","/opt/snaplogic/extensions/components"
it would return:
"/opt/snaplogic/2.0.5","/opt/snaplogic/extensions/components"
if SNAP_HOME=/opt/snaplogic/2.0.5
If OS environment variable SNAP_HOME isn't set,
dot is substituted for SNAP_HOME in string.
@param string_with_vars: String possibly containing parameters
@type string_with_vars: str
"""
# If there is no environment variable SNAP_HOME, assume it's dot.
# This is needed because default values for some of our config
# parameters are defined via SNAP_HOME.
#
# E.g. the default value for the log_dir is now $SNAP_HOME/logs
# (it used to be logs). The QA suite doesn't set that variable.
# So we default it to be backward-compatible.
snap_home = os.environ.get("SNAP_HOME", ".")
# Environment variable may be set up using quotes.
# Remove these, since they aren't legal as part of a path
snap_home = snap_home.replace("\"", "")
# Use python string template to expand SNAP_HOME in string
# using the value of the environment variable SNAP_HOME
template = Template(string_with_vars)
return template.safe_substitute(SNAP_HOME = snap_home)
def parse_size_param(sz_str):
"""
Convert size in terms of byte/mb/kb/gb/tb to a numeric value in bytes.
@param sz_str: The string form of size
@type sz_str: str
@return: The numeric value in terms of bytes.
@rtype: long
"""
if sz_str.isdigit():
sz_num = long(sz_str)
else:
invalid_value = False
try:
sz_str = sz_str.lower()
if sz_str.endswith("kb"):
sz_num = long(sz_str[:-2])
sz_num *= 1024
elif sz_str.endswith("mb"):
sz_num = long(sz_str[:-2])
sz_num *= 1048576
elif sz_str.endswith("gb"):
sz_num = long(sz_str[:-2])
sz_num *= 1073741824L
elif sz_str.endswith("tb"):
# Warning: This may need a python interpreter compiled with special flags
# to handle large files (see http://docs.python.org/lib/posix-large-files.html).
# Also, the OS should be capable of supporting large files.
sz_num = long(sz_str[:-2])
sz_num *= 1099511627776L
else:
return None
except Exception, e:
return None
return sz_num
|