recursively expanding $references in dictionaries

[Alia Khouri]

I was kind of wondering what ways are out there to elegantly expand
'$name' identifiers in nested dictionary value. The problem arose when
I wanted to include that kind of functionality to dicts read from yaml
files such that:

def func(input):
# do something
return output

where:

input = {'firstname': 'John', 'lastname': 'Smith', 'src': 'c:/tmp/
file',
'dir': {'path': '$src', 'fullname': '$firstname $lastname'}}

output = {'firstname': 'John', 'lastname': 'Smith', 'src': 'c:/tmp/
file',
'dir': {'path': 'c:/tmp/file', 'fullname': 'John Smith'}}


Doing this substitution easy done when you have a flat dict, but when
they got nested, I had to resort to an undoubtedly ugly function with
two recursive passes and obvious deficiencies.

Is there a better way to do this?

Thanks for any help...

AK

<code follows>


# test_recurse.py

from string import Template
from pprint import pprint

def expand(dikt):
''''firstname': 'John'}
True
'''
subs = {}
for key, value in dikt.items():
if '$' in value:
subs[key] = Template(value).substitute(dikt)
dikt.update(subs)
return dikt


dikt = {'firstname': 'John', 'lastname': 'Smith',
'fullname': '$firstname $lastname'}

#~ print expand(dikt)

d1 = {'firstname': 'John', 'lastname': 'Smith',
'dir': {'fullname': '$firstname $lastname'}
}
d2 = {'firstname': 'John', 'lastname': 'Smith', 'src': 'c:/tmp/file',
'dir': {'fullname': '$firstname $lastname', 'path':
'$src'}
}

def rexpand(dikt):
subs = {}
names = {}
# pass 1
def recurse(_, sourceDict):
for key, value in sourceDict.items():
if isinstance(value, dict):
recurse({}, value)
elif '$' in value:
subs[key] = value
else:
names[key] = value
recurse({}, dikt)
print 'subs', subs
print 'names', names
print 'dikt (before):', dikt
for key, value in subs.items():
subs[key] = Template(value).substitute(names)
# -----------------------------------------------------
# pass 2
output = {}
def substitute(targetDict, sourceDict):
for key, value in sourceDict.items():
if isinstance(value, dict):
new_target = targetDict.setdefault(key, {})
substitute(new_target, value)
else:
targetDict[key] =
Template(value).substitute(names)
substitute(output, dikt)
print 'output:', output
return output

rexpand(d2)

 

[Alia Khouri]

Oops, I left some redundant cruft in the function... here it is
slightly cleaner:

def expand(dikt):
names = {}
output = {}
def _search(_, sourceDict):
for key, value in sourceDict.items():
if isinstance(value, dict):
_search({}, value)
if not '$' in value:
names[key] = value
_search({}, dikt)
def _substitute(targetDict, sourceDict):
for key, value in sourceDict.items():
if isinstance(value, dict):
new_target = targetDict.setdefault(key, {})
_substitute(new_target, value)
else:
targetDict[key] =
Template(value).substitute(names)
_substitute(output, dikt)
return output

print expand(d2)

 

[Alia Khouri]

Ok. I've reached a nice little conclusion here. Time to go to bed, but
before that I thought I'd share the results (-;

I can now read a yaml file which natively produces a dict tree and
convert it into an object tree with attribute read/write access, dump
that back into a readable yaml string, and then expand references
within that using cheetah in a very nifty self-referential way.*

(see: http://pyyaml.org/wiki/PyYAML) and cheetah templates (http://
www.cheetahtemplate.org/)

Enjoy!


AK


<code below>

from Cheetah.Template import Template
from pprint import pprint
import yaml

class Object(dict):
def __getattr__(self, name):
if name in self: return self[name]
#~ if name in self.__dict__: return getattr(self, name)
def __setattr__(self, name, value):
self[name] = value

def getTree(tree, to_dict=False):
_tree = Object() if not to_dict else dict()
def recurse(targetDict, sourceDict):
for key, value in sourceDict.items():
if isinstance(value, dict):
value = Object(value) if not to_dict else dict(value)
new_target = targetDict.setdefault(key, value)
recurse(new_target, value)
else:
targetDict[key] = value
recurse(_tree, tree)
return _tree


config = '''
app:
name: appname
copyright: me.org 2007

dir:
src: /src/$app.name
'''
# from yml dict tree to obj tree
root = getTree(yaml.load(config))
print root
print
assert root.app.name == root.app['name']
root.app.name = "the_monster"

# from obj tree to dict tree
root = getTree(root, to_dict=True)

# from dict tree to yaml string
s = yaml.dump(root)
print s

# use cheetah templates to expand references
print str(Template(s, searchList=[root]))