NEWBIE: Script help needed


L

Lorenzo

I have this script that I want to use weekly to send me email with
information regarding disk space and available upgrades for my system.
This script is actually a learning tool for me as I learn Python. The
problem I've run into has me stumped and I need some help. What happens
is when the script runs it does these things, parses the result and
appends that to an html string:

1) checks disk space by using df -t reiserfs
2) runs time emerge --sync
3) runs emerge -uvp world
4) runs emerge -uv --fetchonly world

The 'emerge' command is a Gentoo specific one. If I remove step 3),
everything else runs just fine, the email is sent and I receive what I
expect. But when step 3) is allowed to run, even if its the only command
that runs, it hangs somewhere in the function getCommandOutput. If I try
and debug the command, it appears to hang on this line:
err = child.wait()

I suspect a race condition, but I'm not sure how to proceed, can someone
lend me a hand. Here is the script I wrote, I got the command
getCommandOutput from this site:
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
TIA

Code:
#!/usr/bin/python

################################
### NEED TO RUN THIS AS ROOT ###
### EMERGE SYNC REQUIRES THIS ###
################################
import os, re, smtplib, MimeWriter, mimetools, cStringIO, popen2, fcntl, 
select, pdb

cmd = 'df -t reiserfs'
finalList = []
theOutput = []

text = "This realy should be in HTML"


html = "<html>\
<head>\
<meta http-equiv=\"Content-Type\" content=\"text/html; 
charset=iso-8859-1\">\
</head>\
<BODY><BR><font color='green'><em>Disk Utilization on 
Hedley:</em></font><BR>"

out = cStringIO.StringIO()
writer = MimeWriter.MimeWriter(out)
txtin = cStringIO.StringIO(text)


def createhtmlmail (html, text, subject):
         """Create a mime-message that will render HTML in popular
      MUAs, text in better ones"""
   import MimeWriter
   import mimetools
   import cStringIO
   
   out = cStringIO.StringIO() # output buffer for our message 
   htmlin = cStringIO.StringIO(html)
   txtin = cStringIO.StringIO(text)
   
   writer = MimeWriter.MimeWriter(out)
   #
   # set up some basic headers... we put subject here
   # because smtplib.sendmail expects it to be in the
   # message body
   #
   writer.addheader("Subject", subject)
   writer.addheader("MIME-Version", "1.0")

   writer.addheader("From", "[email protected]")
      writer.addheader("To", "[email protected]")
   #
   # start the multipart section of the message
   # multipart/alternative seems to work better
   # on some MUAs than multipart/mixed
   #
   writer.startmultipartbody("alternative")
   writer.flushheaders()
   #
   # the plain text section
   #
   subpart = writer.nextpart()
   subpart.addheader("Content-Transfer-Encoding", "quoted-printable")
   pout = subpart.startbody("text/plain", [("charset", 'us-ascii')])
   mimetools.encode(txtin, pout, 'quoted-printable')
   txtin.close()
   #
   # start the html subpart of the message
   #
   subpart = writer.nextpart()
   subpart.addheader("Content-Transfer-Encoding", "quoted-printable")
   #
   # returns us a file-ish object we can write to
   #
   pout = subpart.startbody("text/html", [("charset", 'us-ascii')])
   mimetools.encode(htmlin, pout, 'quoted-printable')
   htmlin.close()
   #
   # Now that we're done, close our writer and
   # return the message body
   #
   writer.lastpart()
   msg = out.getvalue()
   out.close()
   print msg
   return msg

def makeNonBlocking(fd):
    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
    try:
   fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
    except AttributeError:
   fcntl.fcntl(fd, fcntl.F_SETFL, fl | fcntl.FNDELAY)
    

def getCommandOutput(command):
    theOutput = []
    child = popen2.Popen3(command, 1) # capture stdout and stderr from 
command
    child.tochild.close()             # don't need to talk to child
    outfile = child.fromchild 
    outfd = outfile.fileno()
    errfile = child.childerr
    errfd = errfile.fileno()
    makeNonBlocking(outfd)            # don't deadlock!
    makeNonBlocking(errfd)
    outdata = errdata = ''
    outeof = erreof = 0
    while 1:
   ready = select.select([outfd,errfd],[],[]) # wait for input
   if outfd in ready[0]:
       outchunk = outfile.read()
       if outchunk == '': outeof = 1
       outdata = outdata + outchunk
   if errfd in ready[0]:
       errchunk = errfile.read()
       if errchunk == '': erreof = 1
       errdata = errdata + errchunk
   if outeof and erreof: break
   select.select([],[],[],.1) # give a little time for buffers to fill
   err = child.wait()
    if err != 0: 
   raise RuntimeError, '%s failed w/ exit code %d\n%s' % (command, err, 
errdata)
    theOutput.append(outdata)
    theOutput.append(errdata)
    return theOutput


#Run df and get the disk info
output =  os.popen(cmd)

# match two or more spaces, the header line has a sngle
# space between the 'Mouted on' field
# We need to keep those together
# The other spaces are the separation in the field headers
# To get the output from df down to just the field headers
# and the data, we need to match 2 or more spaces
# -1 eliminates the \n at the end of output.
# We'll get it back when we write each line to the
# mail message, I suspect

html += "<font color='blue'>"

for lines in output.readlines():

   p = re.compile('\ +')
   formattedText = p.subn('      ', lines[:-1], 5) 
   html += formattedText[0] + '<BR>'

html += "</font>"

mydata = getCommandOutput("time emerge --sync")


p = re.compile('\ +')
p.subn(' ', mydata[1])[0]
p = re.compile('\n')
string = (p.subn('<BR>', mydata[1]))[0]

html += "<BR><BR><em><font color='green'>Sync Time: " + "<BR>" + 
"</font></em>" + "<font color='blue'>" + string + "</font>"


mydata = ""
mydata =  getCommandOutput("emerge -uvp world")

p = re.compile('\ +')
(p.subn(' ', mydata[0]))[0]
p = re.compile('\n')
html += "<font color='blue'>" + (p.subn('<BR>', mydata[0]))[0] +  
"</font>"



try:
   getCommandOutput('emerge -uv --fetchonly world')
   html += "<font color='Green'><em><BR>Fetch completed 
successfuly!</em></font>"
except RuntimeError:
      html += "< font color='Red'><BR>******Fetch did not complete 
successfully!*********</font>"

html+= '</BODY> </HTML>'

out = cStringIO.StringIO()
writer = MimeWriter.MimeWriter(out)
txtin = cStringIO.StringIO("This should be in HTML")
subject = "Emerge info and disk utilization on Hedley"
message = createhtmlmail(html, text, subject)
server = smtplib.SMTP("hedley")
server.sendmail('[email protected]', '[email protected]', message)
server.quit()
 
Ad

Advertisements

N

Nick Vatamaniuc

If the other commands work but 3) doesn't, it means there is something
different (wrong?) with the command.

So try running 3) , then one of the other ones and see the difference.


The getCommandOutput() , I suspect, just waits for the data from the
actual command and the command is not returning anything. It could be
because it just takes way too long (I am not familiar with Gentoo, so
not sure if emerge world takes 1 second or 24 hours...) or perhaps the
"emerge -uvp world" stops at some point and is waiting for input (a
command prompt like "are you sure you want to do this [Y/n]?"

For more in depth on subprocesses in Python take a look at the
subprocess module:
http://docs.python.org/lib/module-subprocess.html

Hope this helps,
Nick V.


I have this script that I want to use weekly to send me email with
information regarding disk space and available upgrades for my system.
This script is actually a learning tool for me as I learn Python. The
problem I've run into has me stumped and I need some help. What happens
is when the script runs it does these things, parses the result and
appends that to an html string:

1) checks disk space by using df -t reiserfs
2) runs time emerge --sync
3) runs emerge -uvp world
4) runs emerge -uv --fetchonly world

The 'emerge' command is a Gentoo specific one. If I remove step 3),
everything else runs just fine, the email is sent and I receive what I
expect. But when step 3) is allowed to run, even if its the only command
that runs, it hangs somewhere in the function getCommandOutput. If I try
and debug the command, it appears to hang on this line:
err = child.wait()

I suspect a race condition, but I'm not sure how to proceed, can someone
lend me a hand. Here is the script I wrote, I got the command
getCommandOutput from this site:
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
TIA

Code:
#!/usr/bin/python

################################
### NEED TO RUN THIS AS ROOT ###
### EMERGE SYNC REQUIRES THIS ###
################################
import os, re, smtplib, MimeWriter, mimetools, cStringIO, popen2, fcntl,
select, pdb

cmd = 'df -t reiserfs'
finalList = []
theOutput = []

text = "This realy should be in HTML"


html = "<html>\
<head>\
<meta http-equiv=\"Content-Type\" content=\"text/html;
charset=iso-8859-1\">\
</head>\
<BODY><BR><font color='green'><em>Disk Utilization on
Hedley:</em></font><BR>"

out = cStringIO.StringIO()
writer = MimeWriter.MimeWriter(out)
txtin = cStringIO.StringIO(text)


def createhtmlmail (html, text, subject):
"""Create a mime-message that will render HTML in popular
MUAs, text in better ones"""
import MimeWriter
import mimetools
import cStringIO

out = cStringIO.StringIO() # output buffer for our message
htmlin = cStringIO.StringIO(html)
txtin = cStringIO.StringIO(text)

writer = MimeWriter.MimeWriter(out)
#
# set up some basic headers... we put subject here
# because smtplib.sendmail expects it to be in the
# message body
#
writer.addheader("Subject", subject)
writer.addheader("MIME-Version", "1.0")

writer.addheader("From", "[email protected]")
writer.addheader("To", "[email protected]")
#
# start the multipart section of the message
# multipart/alternative seems to work better
# on some MUAs than multipart/mixed
#
writer.startmultipartbody("alternative")
writer.flushheaders()
#
# the plain text section
#
subpart = writer.nextpart()
subpart.addheader("Content-Transfer-Encoding", "quoted-printable")
pout = subpart.startbody("text/plain", [("charset", 'us-ascii')])
mimetools.encode(txtin, pout, 'quoted-printable')
txtin.close()
#
# start the html subpart of the message
#
subpart = writer.nextpart()
subpart.addheader("Content-Transfer-Encoding", "quoted-printable")
#
# returns us a file-ish object we can write to
#
pout = subpart.startbody("text/html", [("charset", 'us-ascii')])
mimetools.encode(htmlin, pout, 'quoted-printable')
htmlin.close()
#
# Now that we're done, close our writer and
# return the message body
#
writer.lastpart()
msg = out.getvalue()
out.close()
print msg
return msg

def makeNonBlocking(fd):
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
try:
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
except AttributeError:
fcntl.fcntl(fd, fcntl.F_SETFL, fl | fcntl.FNDELAY)


def getCommandOutput(command):
theOutput = []
child = popen2.Popen3(command, 1) # capture stdout and stderr from
command
child.tochild.close()             # don't need to talk to child
outfile = child.fromchild
outfd = outfile.fileno()
errfile = child.childerr
errfd = errfile.fileno()
makeNonBlocking(outfd)            # don't deadlock!
makeNonBlocking(errfd)
outdata = errdata = ''
outeof = erreof = 0
while 1:
ready = select.select([outfd,errfd],[],[]) # wait for input
if outfd in ready[0]:
outchunk = outfile.read()
if outchunk == '': outeof = 1
outdata = outdata + outchunk
if errfd in ready[0]:
errchunk = errfile.read()
if errchunk == '': erreof = 1
errdata = errdata + errchunk
if outeof and erreof: break
select.select([],[],[],.1) # give a little time for buffers to fill
err = child.wait()
if err != 0:
raise RuntimeError, '%s failed w/ exit code %d\n%s' % (command, err,
errdata)
theOutput.append(outdata)
theOutput.append(errdata)
return theOutput


#Run df and get the disk info
output =  os.popen(cmd)

# match two or more spaces, the header line has a sngle
# space between the 'Mouted on' field
# We need to keep those together
# The other spaces are the separation in the field headers
# To get the output from df down to just the field headers
# and the data, we need to match 2 or more spaces
# -1 eliminates the \n at the end of output.
# We'll get it back when we write each line to the
# mail message, I suspect

html += "<font color='blue'>"

for lines in output.readlines():

p = re.compile('\ +')
formattedText = p.subn('      ', lines[:-1], 5)
html += formattedText[0] + '<BR>'

html += "</font>"

mydata = getCommandOutput("time emerge --sync")


p = re.compile('\ +')
p.subn(' ', mydata[1])[0]
p = re.compile('\n')
string = (p.subn('<BR>', mydata[1]))[0]

html += "<BR><BR><em><font color='green'>Sync Time: " + "<BR>" +
"</font></em>" + "<font color='blue'>" + string + "</font>"


mydata = ""
mydata =  getCommandOutput("emerge -uvp world")

p = re.compile('\ +')
(p.subn(' ', mydata[0]))[0]
p = re.compile('\n')
html += "<font color='blue'>" + (p.subn('<BR>', mydata[0]))[0] +
"</font>"



try:
getCommandOutput('emerge -uv --fetchonly world')
html += "<font color='Green'><em><BR>Fetch completed
successfuly!</em></font>"
except RuntimeError:
html += "< font color='Red'><BR>******Fetch did not complete
successfully!*********</font>"

html+= '</BODY> </HTML>'

out = cStringIO.StringIO()
writer = MimeWriter.MimeWriter(out)
txtin = cStringIO.StringIO("This should be in HTML")
subject = "Emerge info and disk utilization on Hedley"
message = createhtmlmail(html, text, subject)
server = smtplib.SMTP("hedley")
server.sendmail('[email protected]', '[email protected]', message)
server.quit()
 
Ad

Advertisements

L

Lorenzo Thurman

Thanks for the reply, but I know there is something wrong with the
command, I'm just not sure how to troubleshoot it. Anyway, I can execute
all of the commands from the command line, but only 3 fails when run
from within the script. I'll take a look at the link.


Nick said:
If the other commands work but 3) doesn't, it means there is something
different (wrong?) with the command.

So try running 3) , then one of the other ones and see the difference.


The getCommandOutput() , I suspect, just waits for the data from the
actual command and the command is not returning anything. It could be
because it just takes way too long (I am not familiar with Gentoo, so
not sure if emerge world takes 1 second or 24 hours...) or perhaps the
"emerge -uvp world" stops at some point and is waiting for input (a
command prompt like "are you sure you want to do this [Y/n]?"

For more in depth on subprocesses in Python take a look at the
subprocess module:
http://docs.python.org/lib/module-subprocess.html

Hope this helps,
Nick V.


I have this script that I want to use weekly to send me email with
information regarding disk space and available upgrades for my system.
This script is actually a learning tool for me as I learn Python. The
problem I've run into has me stumped and I need some help. What happens
is when the script runs it does these things, parses the result and
appends that to an html string:

1) checks disk space by using df -t reiserfs
2) runs time emerge --sync
3) runs emerge -uvp world
4) runs emerge -uv --fetchonly world

The 'emerge' command is a Gentoo specific one. If I remove step 3),
everything else runs just fine, the email is sent and I receive what I
expect. But when step 3) is allowed to run, even if its the only command
that runs, it hangs somewhere in the function getCommandOutput. If I try
and debug the command, it appears to hang on this line:
err = child.wait()

I suspect a race condition, but I'm not sure how to proceed, can someone
lend me a hand. Here is the script I wrote, I got the command
getCommandOutput from this site:
http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52296
TIA

Code:
#!/usr/bin/python

################################
### NEED TO RUN THIS AS ROOT ###
### EMERGE SYNC REQUIRES THIS ###
################################
import os, re, smtplib, MimeWriter, mimetools, cStringIO, popen2, fcntl,
select, pdb

cmd = 'df -t reiserfs'
finalList = []
theOutput = []

text = "This realy should be in HTML"


html = "<html>\
<head>\
<meta http-equiv=\"Content-Type\" content=\"text/html;
charset=iso-8859-1\">\
</head>\
<BODY><BR><font color='green'><em>Disk Utilization on
Hedley:</em></font><BR>"

out = cStringIO.StringIO()
writer = MimeWriter.MimeWriter(out)
txtin = cStringIO.StringIO(text)


def createhtmlmail (html, text, subject):
"""Create a mime-message that will render HTML in popular
MUAs, text in better ones"""
import MimeWriter
import mimetools
import cStringIO

out = cStringIO.StringIO() # output buffer for our message
htmlin = cStringIO.StringIO(html)
txtin = cStringIO.StringIO(text)

writer = MimeWriter.MimeWriter(out)
#
# set up some basic headers... we put subject here
# because smtplib.sendmail expects it to be in the
# message body
#
writer.addheader("Subject", subject)
writer.addheader("MIME-Version", "1.0")

writer.addheader("From", "[email protected]")
writer.addheader("To", "[email protected]")
#
# start the multipart section of the message
# multipart/alternative seems to work better
# on some MUAs than multipart/mixed
#
writer.startmultipartbody("alternative")
writer.flushheaders()
#
# the plain text section
#
subpart = writer.nextpart()
subpart.addheader("Content-Transfer-Encoding", "quoted-printable")
pout = subpart.startbody("text/plain", [("charset", 'us-ascii')])
mimetools.encode(txtin, pout, 'quoted-printable')
txtin.close()
#
# start the html subpart of the message
#
subpart = writer.nextpart()
subpart.addheader("Content-Transfer-Encoding", "quoted-printable")
#
# returns us a file-ish object we can write to
#
pout = subpart.startbody("text/html", [("charset", 'us-ascii')])
mimetools.encode(htmlin, pout, 'quoted-printable')
htmlin.close()
#
# Now that we're done, close our writer and
# return the message body
#
writer.lastpart()
msg = out.getvalue()
out.close()
print msg
return msg

def makeNonBlocking(fd):
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
try:
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NDELAY)
except AttributeError:
fcntl.fcntl(fd, fcntl.F_SETFL, fl | fcntl.FNDELAY)


def getCommandOutput(command):
theOutput = []
child = popen2.Popen3(command, 1) # capture stdout and stderr from
command
child.tochild.close()             # don't need to talk to child
outfile = child.fromchild
outfd = outfile.fileno()
errfile = child.childerr
errfd = errfile.fileno()
makeNonBlocking(outfd)            # don't deadlock!
makeNonBlocking(errfd)
outdata = errdata = ''
outeof = erreof = 0
while 1:
ready = select.select([outfd,errfd],[],[]) # wait for input
if outfd in ready[0]:
outchunk = outfile.read()
if outchunk == '': outeof = 1
outdata = outdata + outchunk
if errfd in ready[0]:
errchunk = errfile.read()
if errchunk == '': erreof = 1
errdata = errdata + errchunk
if outeof and erreof: break
select.select([],[],[],.1) # give a little time for buffers to fill
err = child.wait()
if err != 0:
raise RuntimeError, '%s failed w/ exit code %d\n%s' % (command, err,
errdata)
theOutput.append(outdata)
theOutput.append(errdata)
return theOutput


#Run df and get the disk info
output =  os.popen(cmd)

# match two or more spaces, the header line has a sngle
# space between the 'Mouted on' field
# We need to keep those together
# The other spaces are the separation in the field headers
# To get the output from df down to just the field headers
# and the data, we need to match 2 or more spaces
# -1 eliminates the \n at the end of output.
# We'll get it back when we write each line to the
# mail message, I suspect

html += "<font color='blue'>"

for lines in output.readlines():

p = re.compile('\ +')
formattedText = p.subn('      ', lines[:-1], 5)
html += formattedText[0] + '<BR>'

html += "</font>"

mydata = getCommandOutput("time emerge --sync")


p = re.compile('\ +')
p.subn(' ', mydata[1])[0]
p = re.compile('\n')
string = (p.subn('<BR>', mydata[1]))[0]

html += "<BR><BR><em><font color='green'>Sync Time: " + "<BR>" +
"</font></em>" + "<font color='blue'>" + string + "</font>"


mydata = ""
mydata =  getCommandOutput("emerge -uvp world")

p = re.compile('\ +')
(p.subn(' ', mydata[0]))[0]
p = re.compile('\n')
html += "<font color='blue'>" + (p.subn('<BR>', mydata[0]))[0] +
"</font>"



try:
getCommandOutput('emerge -uv --fetchonly world')
html += "<font color='Green'><em><BR>Fetch completed
successfuly!</em></font>"
except RuntimeError:
html += "< font color='Red'><BR>******Fetch did not complete
successfully!*********</font>"

html+= '</BODY> </HTML>'

out = cStringIO.StringIO()
writer = MimeWriter.MimeWriter(out)
txtin = cStringIO.StringIO("This should be in HTML")
subject = "Emerge info and disk utilization on Hedley"
message = createhtmlmail(html, text, subject)
server = smtplib.SMTP("hedley")
server.sendmail('[email protected]', '[email protected]', message)
server.quit()
 

Top