__slots__

[flori]

i try to greate somthing like this

class ca(object): __slots__ = ("a",)
class cb(ca): __slots__ = ("a","b")
class cc(ca): __slots__ = ("a","c")
class cd(cb,cc): __slots__ = ("a","b","c","d")

but i didn't find a simple solution
so i'm using a metaclass that generate a __slots__-free and a
__slots__-version
the __slots__ -free is returned
the __new__ of the base class look for the sloted verion
the classes defines a (class)-attribute __object_slots__ with their
new attribute names.

killer? someone a good idea?

my soure (something not important i'm cutting away)

import sys
import types
import inspect
import weakref
#import
#
class const(object):
__slots__=()
cs_classid = intern("_classid")
cs__all_persitent_slots__ = intern("__all_persitent_slots__")
cs__all_nonpersitent_slots__ =
intern("__all_nonpersitent_slots__")
cs__all_slots__ = intern("__all_slots__")
cs__cls_unsloted__ = intern("__cls_unsloted__")
cs__nonpersitent_slots__ = intern("__nonpersitent_slots__")
cs__object_slots__ = intern("__object_slots__")
cs__object_slots_nomore__ = intern("__object_slots_nomore__")
cs__slots__ = intern("__slots__")
cs__slots_fixed__ = intern("__slots_fixed__")
cs_lstobjProperty = intern("_lstobjProperty")
cslstSlots = intern("lstSlots")
cs_sloted_prefix = intern("_sloted_")
cslstNonPersitentSlots = intern("lstNonPersitentSlots")
#
#
class ClCustomBase(object):
""" no really need for this but i like it
may be i need for debugging, if..
"""
pass
#
class ClMetaClass(ClCustomBase):
#
def __new__(cls, sName, tupBases, dctClassDefinition):
""" calculates the class-attribute.
calcs
__all_slots__ := __object_slots__ + (for all
tupBases).__all_slots__
__all_nonpersitent_slots__ := __nonpersitent_slots__ +
(for all tupBases).__all_nonpersitent_slots__
__all_persitent_slots__ := __all_slots__ -
__all_nonpersitent_slots__
"""
# init
objClsUnsloted=None
dctDefinitionLocals = sys._getframe(1).f_locals
# create a (hopefully) unique name
# first i try to use the id(objClsUnsloted) but it's not known
before creation
# - and i should know it before
dctClassDefinition[const.cs_classid] = sName +
hex(id(tupBases) ^ id(sys._getframe(0)) ) ^
id(dctClassDefinition.get(const.cs__object_slots__, None))
dctAddInfo = {
const.cslstSlots: list(),
const.cslstNonPersitentSlots: list(),
}
# prepare definition of unsloted
cls.calculateSlots(sName, tupBases, dctClassDefinition,
dctAddInfo, dctDefinitionLocals)
objClsUnsloted = type(sName, tupBases, dctClassDefinition)
#
# prepare definition of sloted
dctClassDefinition[const.cs__slots__] =
tuple(dctAddInfo[const.cslstSlots])
dctClassDefinition[const.cs__cls_unsloted__] = objClsUnsloted
# definition of the sloted class
objClsSloted = type(const.cs_sloted_prefix+sName,
(objClsUnsloted,), dctClassDefinition)
objClsUnsloted.__cls_sloted__ = objClsSloted
dctDefinitionLocals[const.cs_sloted_prefix+sName] =
objClsSloted
# done
dctDefinitionLocals = None # help gc
return objClsUnsloted
#
def __init__(self, sName, tupBases, dctClassDefinition):
super(ClMetaBase,self).__init__(sName, tupBases,
dctClassDefinition)
#
def calculateSlots(cls, sName, tupBases, dctClassDefinition,
dctAddInfo, dctDefinitionLocals):
""" calculates all slot things needed fo the sloted version -
ah you geuess it
"""
lstSlots=dctAddInfo[const.cslstSlots]
tupClassSlots =
dctClassDefinition.get(const.cs__object_slots__, None)
if isinstance(tupClassSlots, basestring):
raise TypeError, "in class "+sName+":__object_slots__ is
defined as string (may be you forgotten a ,?)"
if tupClassSlots is None:
# or should we simply pass ?
raise TypeError, "The class "+sName+" (with ClBase as
superclass) must have a __object_slots__-attribute like
\n__object_slots__ = "+str(tuple(lstSlots))+".\n"+", but
-"+str(tupClassSlots)+"- found \n"
else:
lstSlots.extend( tupClassSlots )
#
lstNonPersitentSlots=dctAddInfo[const.cslstNonPersitentSlots]
lstNonPersitentSlots.extend(
dctClassDefinition.get(const.cs__nonpersitent_slots__, ()) )
#
for objCls in tupBases:
tupSubSlots = getattr(objCls, const.cs__all_slots__, None)
if tupSubSlots is not None:
lstSlots.extend(tupSubSlots)
#
tupSubSlots = getattr(objCls,
const.cs__nonpersitent_slots__, None)
if tupSubSlots is not None:
lstNonPersitentSlots.extend(tupSubSlots)
#no more needed
#lstBasesUnsloted.append(getattr(objCls,
cs__cls_unsloted__, objCls))
#
#no more needed
#dctAddInfo["tupBasesUnsloted"] = tuple( lstBasesUnsloted )
#add nonpersitent slots to the all slots
#
lstNonPersitentSlots.sort()
for iIndex in xrange(len(lstNonPersitentSlots)-2, -1, -1):
if lstNonPersitentSlots[iIndex] ==
lstNonPersitentSlots[iIndex+1]:
del lstNonPersitentSlots[iIndex+1]
#
#
tupObjectSlotsNoMore =
dctClassDefinition.get(const.cs__object_slots_nomore__, None)
if tupObjectSlotsNoMore is not None:
for sObjectSlotsNoMore in tupObjectSlotsNoMore:
try:
lstNonPersitentSlots.remove(sObjectSlotsNoMore)
except ValueError:
pass
#
#
#
lstSlots.extend(lstNonPersitentSlots)
#sort slots and remove doubles
lstSlots.sort()
for iIndex in xrange(len(lstSlots)-2, -1, -1):
s=intern(lstSlots[iIndex])
if s == lstSlots[iIndex+1]:
del lstSlots[iIndex+1]
else:
lstSlots[iIndex]=s
#
#
if tupObjectSlotsNoMore is not None:
for sObjectSlotsNoMore in tupObjectSlotsNoMore:
try:
lstSlots.remove(sObjectSlotsNoMore)
except ValueError:
pass
#
#
#
# non persitent and persitent are disjunct ==> persitent :=
slots - non persitenten
lstPersitentSlots=lstSlots[:]
for sKey in lstNonPersitentSlots:
lstPersitentSlots.remove(sKey)
#
# update class definition
dctClassDefinition[const.cs__all_slots__]=tuple(lstSlots)
dctClassDefinition[const.cs__all_nonpersitent_slots__]=tuple(lstNonPersitentSlots)
dctClassDefinition[const.cs__all_persitent_slots__]=tuple(lstPersitentSlots)
#
# normally there isn't a __slots__ - definition, but if we
cannot trust it
try:
dctClassDefinition.remove(const.cs__slots__)
except:
pass
dctDefinitionLocals = None # help gc
#
calculateSlots=classmethod(calculateSlots)
#
#
class ClBase(ClCustomBase):
""" base of all
"""
#
__metaclass__ = ClMetaClass
__object_slots__ = ()
#__object_slots__ = ("gna", "__weakref__", )
#__weakref__ = 1
#
def __new__(cls, *lstArgs, **kwArgs):
self = super(ClBase,
cls).__new__(getattr(cls,"__cls_sloted__",cls))
return self
#now __init__ is called
# / __new__
def __init__(self, *lstArgs, **kwArgs):
super(ClBase, self).__init__()
#
#

 

[Alex Martelli]

i try to greate somthing like this

class ca(object): __slots__ = ("a",)
class cb(ca): __slots__ = ("a","b")
class cc(ca): __slots__ = ("a","c")
class cd(cb,cc): __slots__ = ("a","b","c","d")

but i didn't find a simple solution

....for WHAT problem...?

slots are inherited... is this perhaps what was unclear to you?


Alex

 

[Ulrich Petri]

i try to greate somthing like this

class ca(object): __slots__ = ("a",)
class cb(ca): __slots__ = ("a","b")
class cc(ca): __slots__ = ("a","c")
class cd(cb,cc): __slots__ = ("a","b","c","d")

but i didn't find a simple solution
so i'm using a metaclass that generate a __slots__-free and a
__slots__-version
the __slots__ -free is returned
the __new__ of the base class look for the sloted verion
the classes defines a (class)-attribute __object_slots__ with their
new attribute names.

killer? someone a good idea?

my soure (something not important i'm cutting away)

<snip lots of code that looks like java in python>

What exactly are you trying to accomplish?

Ciao Ulrich

 

[flori]

i try to greate somthing like this

sorry for silly question
my prob is when i try to do this
class ca(object): __slots__ = ("a",)
class cb(ca): __slots__ = ("a","b")
class cc(ca): __slots__ = ("a","c")
class cd(cb,cc): __slots__ = ("a","b","c","d")

in line "class cd(.." thorws a exception
TypeError: multiple bases have instance lay-out conflict

(if cb (or cc) doesn't define __slots__ it works.)

My only solution is to generate a "unsloted" class (_ca)
and genererate a "sloted" class (ca) with only the __slots__ attribute.
when i had to inherit the class i use the "unsloted" class.

like that:
class _ca(object): pass
class ca(object): __slots__ = ("a",)
class _cb(_ca): pass
class cb(_cb): __slots__ = ("a","b")
class _cc(_ca): pass
class cc(_cc): __slots__ = ("a","c")
class _cd(_cb,_cc): pass
class cd(_cd): __slots__ = ("a","b","c","d")

my source does this a metaclass

my question is is their a nicer way to do this?

 

[Francis Avila]

(e-mail address removed) (flori) wrote in message
my question is is their a nicer way to do this?

Yes. Don't use slots! Why are you using slots?!

 

[Alex Martelli]

flori wrote:
...

class _ca(object): pass
class ca(object): __slots__ = ("a",)
class _cb(_ca): pass
class cb(_cb): __slots__ = ("a","b")
class _cc(_ca): pass
class cc(_cc): __slots__ = ("a","c")
class _cd(_cb,_cc): pass
class cd(_cd): __slots__ = ("a","b","c","d")

my source does this a metaclass

my question is is their a nicer way to do this?

Yes: and the nicer way to get exactly the same result is,
remove all those useless __slots__ declarations. What do
you think they're buying you...?

Maybe you haven't tried the elementary test...:

x = cd()
x.pippo = 'ma va!'
print x.pippo

....?

Once _any_ base class allows instances to have a per-instance
dictionary (which is the case for all classes _except_ the very
special ones where we insert the very-special-purpose and often-
misunderstood *optimization* '__slots__'....), then of course
*all* instances of that class (and an instance of a subclass is
an instance of the base class too) will have a per-instance
dictionary. And once you have a per-instance dictionary anyway,
adding __slots__ is just silly -- can't make the per-inst dict
disappear, so no memory is actually saved, no attribute setting
is prohibited, etc, etc.

And yes, you can't multiply inherit from several classes that
define __slots__ (not even identical slots, if I recall correctly),
just like you can't multiply inherit from different built-in
types (with non-null instance-layout constraints), nor even from
one built-in type and a separate class with __slots__. Defining
__slots__ does constrain the instance layout, and generally speaking
Python doesn't even _try_ to sort out any case of multiple separate
constraints on instance layout.

__slots__ is meant as a special-purpose memory optimization to
save the per-instance memory cost of a dictionary when you're
defining small classes of which a huge number of instances are
meant to exist at the same time, so that the several tens of bytes
per instance that a dictionary might cost is an unacceptable price
to pay compared to the "actual" footprint of each of the huge
number of instances. For this use case, which is really the only
one that __slots__ is intended for, multiple inheritance isn't
really indispensable. All in all, I doubt, therefore, that these
limitations will go away.


Alex

 

[Brad Clements]

_

__slots__ is meant as a special-purpose memory optimization to
save the per-instance memory cost of a dictionary when you're
defining small classes of which a huge number of instances are
meant to exist at the same time, so that the several tens of bytes
per instance that a dictionary might cost is an unacceptable price
to pay compared to the "actual" footprint of each of the huge
number of instances.

And I'd like to add.. I was starting a new embedded Python project when
__slots__ came out, the documentation at the time didn't make clear their
intended use of storage optimization. Rather what *I* read into it was a
way to reduce the possibility of error by mistyping an instance attribute.
Now, why I thought this was neat is beyond me, since in my vast 4 years of
Python programming I've never really had that problem anyway.

But nonetheless I slotted myself into a corner anyway. Now I have to undo
all those nasty slots statements.

My advice.. forget __slots__!

 

[Patrick Maupin]

But nonetheless I slotted myself into a corner anyway. Now I have to undo
all those nasty slots statements.

My advice.. forget __slots__!

In general, I agree. However, if one is contemplating
performance optimizations by "Pyrexcizing" Python code
and creating new embedded types, one might find that the
use of __slots__ is a viable small step on the path to
"cdef object." (If one is the sort who wants to keep testing
and debugging in actual Python instead of Pyrex until the
very last minute.)

One might also find, while taking this path, that the
use of __slots__ by itself makes SOME applications run
fast enough without going all the way to Pyrex. At this
point, one might reasonably conclude (despite the
oft-repeated claim that __slots__ are an un-Pythonic
"premature optimization") that the ability to avoid writing
a C extension by merely adding a single line of (basically
non-executable) code to a class or two is unalloyed goodness.

And in the spare time one creates by not writing a C extension,
one may be forgiven for contemplating whether the original
"premature optimization" claim referred to the code which USES
__slots__ (as some seem to interpret), or to the language feature
of __slots__ itself, which is admittedly not as nice as something
like Psyco has the potential to be, but which nonetheless has the
same advantage over some future optimizations that list
comprehensions have over generator expressions, namely that it
is available NOW, in a working and seemingly stable interpreter.

Regards,
Pat

 

[Mirko Zeibig]

One might also find, while taking this path, that the
use of __slots__ by itself makes SOME applications run
fast enough without going all the way to Pyrex. At this
point, one might reasonably conclude (despite the
oft-repeated claim that __slots__ are an un-Pythonic
"premature optimization") that the ability to avoid writing
a C extension by merely adding a single line of (basically
non-executable) code to a class or two is unalloyed goodness.

Just as an info: I ran some tests using slots vs. "normal classes" vs. tuples,
here are the definitions for the classes:

class NoSlots(object):
def __init__(self, *args):
self.id, self.weight = args

class Slots(object):
__slots__ = ('id', 'weight')
def __init__(self, *args):
self.id, self.weight = args

def createtuple(id, weight):
return (id, weight)

def create(func, count):
w = timer.create("creating %s" % func)
l = [ func(str(id), id*2) for id in xrange(count) ]
w.stop()
showMemUsage(func)
return l

def usetupledirectly(func, count):
w = timer.create("creating %s" % func)
l = [ (str(id), id*2) for id in xrange(count) ]
w.stop()
showMemUsage(func)
return l


created lot's of these and hold them in a list. I measured memory-usage very
roughly using /proc/self/status (VmRSS):

[mize@lxmize python]$ for i in 1000 10000 100000 ; do ./memusage_speed_tuple_vs_simple_classes.py $i; done
********************************************************************************
Creating 1000 objects
MemUsage <class '__main__.NoSlots'>: 2780 kB
MemUsage <class '__main__.Slots'>: 80 kB
MemUsage <function createtuple at 0x8242214>: 88 kB
MemUsage <function usetupledirectly at 0x8214a84>: 88 kB
Runtime UsrTim SysTim Calls NestC PendC TimerName
0.216 0.130 0.000 1 0 0 total
0.039 0.040 0.000 1 0 0 creating <class '__main__.NoSlots'>
0.035 0.040 0.000 1 0 0 creating <class '__main__.Slots'>
0.019 0.020 0.000 1 0 0 creating <function createtuple at 0x8242214>
0.011 0.010 0.000 1 0 0 creating <function usetupledirectly at 0x8214a84>
0.007 0.010 0.000 1 0 0 deleting stuff, disablegc=0
********************************************************************************
Creating 10000 objects
MemUsage <class '__main__.NoSlots'>: 4752 kB
MemUsage <class '__main__.Slots'>: 800 kB
MemUsage <function createtuple at 0x8242214>: 860 kB
MemUsage <function usetupledirectly at 0x8214a84>: 864 kB
Runtime UsrTim SysTim Calls NestC PendC TimerName
1.351 1.230 0.020 1 0 0 total
0.438 0.440 0.000 1 0 0 creating <class '__main__.NoSlots'>
0.365 0.370 0.000 1 0 0 creating <class '__main__.Slots'>
0.186 0.180 0.000 1 0 0 creating <function createtuple at 0x8242214>
0.148 0.130 0.020 1 0 0 creating <function usetupledirectly at 0x8214a84>
0.100 0.100 0.000 1 0 0 deleting stuff, disablegc=0
********************************************************************************
Creating 100000 objects
MemUsage <class '__main__.NoSlots'>: 25160 kB
MemUsage <class '__main__.Slots'>: 8528 kB
MemUsage <function createtuple at 0x8242214>: 9308 kB
MemUsage <function usetupledirectly at 0x8214a84>: 9308 kB
Runtime UsrTim SysTim Calls NestC PendC TimerName
16.979 16.230 0.610 1 0 0 total
5.448 5.230 0.200 1 0 0 creating <class '__main__.NoSlots'>
4.792 4.640 0.140 1 0 0 creating <class '__main__.Slots'>
3.111 2.970 0.140 1 0 0 creating <function createtuple at 0x8242214>
2.805 2.670 0.130 1 0 0 creating <function usetupledirectly at 0x8214a84>
0.716 0.710 0.000 1 0 0 deleting stuff, disablegc=0
[mize@lxmize python]$

So while there is a big memory saving in using __slots__, the time savings I
observed are relatively small. Test system is a Pentium Celeron 500MHz with 256
MB running the Python 2.2 on Linux. Of course never trust measurements you
didn't manipulate yourself ;-).

Regards
Mirko

 

[Patrick Maupin]

So while there is a big memory saving in using __slots__, the time savings I
observed are relatively small. Test system is a Pentium Celeron 500MHz with 256
MB running the Python 2.2 on Linux. Of course never trust measurements you
didn't manipulate yourself ;-).

Always excellent advice, which I usually attempt to follow. I think
I tried to make it fairly clear in the post you were responding to that
__slots__ would not always give great results. However, some empirical
evidence I have (aka measurements I manipulated myself shows that in
_some_ cases the time savings can be substantial. In the cases I have
seen, the time savings actually follow the memory savings. In other
words, if __slots__ can make the difference between paging or not, or
even, possibly, lots of RAM cache misses or not, then it could be
worthwhile. (As your detailed post makes clear, saving 70% of the RAM
by using __slots__ on simple objects is not out of the question, although
I do find it somewhat surprising that on your small objects the __slots__
requirement was even smaller than a tuple!)

Also, if I am not misremembering (it was quite a few months back),
I think the speed benefit was more pronounced in 2.3 than in 2.2,
so maybe there are some additional interpreter optimizations there.

FWIW, my use case for __slots__ involved creating at least tens of
millions of objects, and I was not exaggerating when I wrote that
I was on the way to writing a C extension (in Pyrex).

Regards,
Pat