# $SnapHashLicense:
#
# SnapLogic - Open source data services
#
# Copyright (C) 2008, SnapLogic, Inc. All rights reserved.
#
# See http://www.snaplogic.org for more information about
# the SnapLogic project.
#
# This program is free software, distributed under the terms of
# the GNU General Public License Version 2. See the LEGAL file
# at the top of the source tree.
#
# "SnapLogic" is a trademark of SnapLogic, Inc.
#
#
# $
# $Id: memory.py 3388 2008-06-19 21:11:37Z kurt $
"""
Memory utilities for SnapStream internal use.
"""
from decimal import Decimal
from datetime import datetime
from snaplogic.common.snap_exceptions import *
DEFAULT_BLOCK_SIZE = 10240
"""
Default block sized used by many binary SnapStream objects such as SelectableBinaryPipe.
"""
MAX_BUFFER_SIZE = None
"""
Maximum size of buffers used for SnapStream objects. This size is a per-object limit not a global
limit. For HTTP connections internal to the CC where both endpoints are within the same CC process,
it is likely the buffers used will total up to 2*MAX_BUFFER_SIZE or more bytes.
In order to simplify implementation, this value is not the exact limit. The buffers will stop growing once
their size is greater than or equal to this size. For record streams, this means an extra record could
occur in the buffer before it blocks. For binary streams, a full block read extra could occur.
The value is set at startup by reading the max_stream_buffer_size config entry from the CC section. A default
value is provided by the config object if not found.
"""
def estimate_size(value):
"""
Estimate the size (in bytes) of the value passed in.
Python does not give us any direct indication of memory size of objects. Even the size used by standard
types can vary between implementations and architectures. This function applies a heuristic to estimate
the size. No guarantee on the error of this function exists, but it should provide a reasonable comparison
between objects and their size to be used for buffer sizes.
Only some classes (those expected in a SnapStream) are supported here.
@param value: A python value or object.
@type value: any
@return: Estimated size of value in bytes.
@rtype: int
"""
if type(value) is int or value is None:
return 4
elif type(value) is long:
return 8
elif type(value) in [str, unicode]:
return len(value)
elif type(value) is Decimal:
return estimate_size(value.as_tuple())
elif type(value) is datetime:
return estimate_size(tuple(value.timetuple()))
elif type(value) in [list, tuple]:
size = 0
for elem in value:
size += estimate_size(elem)
return size
else:
raise SnapObjTypeError("Unexpected object of type " + str(type(value)))
|