S
Steven W. Orr
I'm writing a program that uses paramiko to run a lot of commands over ssh.
Some of the commands take time to run and they write to stdout and stderr as a
normal part of their operation so that we can see progress happening.
I can't seem to get the output from the remote commands (which is input to me)
to be line buffered. If I run the commands using ssh, they line buffer nicely.
If I run them through paramiko, they end up fully buffered.
I stole this code I found as a base. The code that I got looked like the
execute_capture method (below). I added the execute method so that I could
poll for the result from both stderr and stdout. Note that I am calling
channel.get_pty, but that doesn't change the fact that the results are not
line buffered.
Can anyone suggest a way to solve this?
The code I'm using follows:
#! /usr/bin/python
"""
Friendly Python SSH2 interface using paramiko
"""
import os
import sys
import tempfile
import paramiko
import select
from collections import namedtuple
ExecStatus = namedtuple('ExecStatus', 'status stdout stderr')
class Connection(object):
"""
Connects and logs into the specified hostname.
Arguments that are not given are guessed from the environment.
"""
def __init__(self,
host,
username = None,
private_key = None,
password = None,
port = 22,
blocking_cmds = True,
verbose = False,
):
self._sftp_live = False
self._sftp = None
if not username:
username = os.environ['LOGNAME']
# Log to a temporary file if requested.
if verbose:
self.templog = tempfile.mkstemp('.txt', 'ssh-')[1]
paramiko.util.log_to_file(self.templog)
else:
self.templog = False
# Begin the SSH transport.
self._transport = paramiko.Transport((host, port))
self._tranport_live = True
# Authenticate the transport.
if password:
# Using Password.
self._transport.connect(username = username, password = password)
else:
# Use Private Key.
if not private_key:
# Try to use default key.
if os.path.exists(os.path.expanduser('~/.ssh/id_rsa')):
private_key = '~/.ssh/id_rsa'
elif os.path.exists(os.path.expanduser('~/.ssh/id_dsa')):
private_key = '~/.ssh/id_dsa'
else:
raise TypeError, "You have not specified a password or key."
private_key_file = os.path.expanduser(private_key)
rsa_key = paramiko.RSAKey.from_private_key_file(private_key_file)
self._transport.connect(username = username, pkey = rsa_key)
def _sftp_connect(self):
"""Establish the SFTP connection."""
if not self._sftp_live:
self._sftp = paramiko.SFTPClient.from_transport(self._transport)
self._sftp_live = True
def get(self, remotepath, localpath = None):
"""Copies a file between the remote host and the local host."""
if not localpath:
localpath = os.path.split(remotepath)[1]
self._sftp_connect()
self._sftp.get(remotepath, localpath)
def put(self, localpath, remotepath = None):
"""Copies a file between the local host and the remote host."""
if not remotepath:
remotepath = os.path.split(localpath)[1]
self._sftp_connect()
self._sftp.put(localpath, remotepath)
def execute(self, command):
"""
Execute the given commands on a remote machine.
Return value is exit status of the remote command.
"""
# This execute method is similar to execute_capture below. The
# difference is that this method gets the stdout and stderr of
# the runnning command and forwards it on to the correct
# channel within this process.
# To do this, we use the poll(2) system call which comes from
# the select package.
def _write(fd, chan, syschan):
"""
_write internal method to check an fd from the list of fds
for a POLLIN event, read the data that's there, and if there's
anything there, then write it to the correct channel.
Return True if there was something to read.
"""
ret = False
if fd[1] & select.POLLIN:
if fd[0] == chan.channel.fileno():
ss = chan.readline()
ret = len(ss) != 0
if ret:
# No need to strip and then print with a newline.
# because every line is newline terminated.
print >> syschan, ss[:-1]
return ret
# Open a channel of type session. Same as open_channel('session')
channel = self._transport.open_session()
# Calling get_pty does get us a pty
# If the 2nd arg is 1, then it should be line buffered. Apparently,
# in this context, line buffering refers to output to the process,
# not input from the process.
channel.get_pty()
# Run command on the session of type channel. This returns immediately.
channel.exec_command(command)
# Get the stdout and stderr file descriptors. In this context, makefile
# has nothing to do with the make utility. It's about making
# file descriptors.
stdout = channel.makefile('rb', 1)
stderr = channel.makefile_stderr('rb', 1)
# Create a polling object. We really only care about POLLIN events.
po = select.poll()
po.register(stdout.channel, select.POLLIN)
po.register(stderr.channel, select.POLLIN)
# Set found_output tto True to start the loop. Set it to False
# as an initial value inside the loop, and OR the value on if any data
# got written by either channel.
found_output = True
while found_output == True:
found_output = False
fds = po.poll()
for fd in fds:
found_output |= _write(fd, stdout, sys.stdout)
found_output |= _write(fd, stderr, sys.stderr)
status = channel.recv_exit_status()
channel.close()
return status
def execute_capture(self, command):
"""
Execute the given commands on a remote machine.
Return value is a 3-tuple: exit status, stdout and stderr
Output is not written out.
"""
# Open a channel of type session. Same as open_channel('session')
channel = self._transport.open_session()
# Run command on the session of type channel. This returns immediately.
channel.exec_command(command)
# Collect stdout and stderr into lists.
# The alternative would be to harvest the 3-tuple std{in,out,err}
# of file descriptors returned by exec_command.
stdout = channel.makefile('rb', -1).readlines()
stderr = channel.makefile_stderr('rb', -1).readlines()
# Not well documented, but recv_exit_status will block until
# the command completes. Return value is the exit status.
status = channel.recv_exit_status()
channel.close()
return ExecStatus(status, stdout, stderr)
def close_transport(self):
# Close the SSH Transport.
if self._tranport_live:
self._transport.close()
self._tranport_live = False
def close_sftp(self):
# Close SFTP Connection.
if self._sftp_live:
self._sftp.close()
self._sftp_live = False
def close(self):
"""Closes the connection and cleans up."""
self.close_transport()
self.close_sftp()
# start the ball rolling.
if __name__ == "__main__":
def proc_exec(cmd):
status = myssh.execute(cmd)
print "Just ran %s with exit status %d"%(cmd, status)
# Another choice is to call
# def proc_exec(cmd):
# run = myssh.execute_capture(cmd)
# print "Just ran %s with exit status %d"%(cmd, status)
# if run.stdout:
# print "output:\n", ''.join(run.stdout)[:-1]
# if run.stderr:
# print "error:\n", ''.join(run.stderr)[:-1]
# Little test when called directly.
# Set these to your own details.
myssh = Connection('host', username='user')
proc_exec('pwd')
proc_exec('false')
proc_exec('xxx.sh')
proc_exec('cd sorr; pwd')
proc_exec('pwd')
proc_exec('tty')
myssh.close()
--
Time flies like the wind. Fruit flies like a banana. Stranger things have .0.
happened but none stranger than this. Does your driver's license say Organ ..0
Donor?Black holes are where God divided by zero. Listen to me! We are all- 000
individuals! What if this weren't a hypothetical question?
steveo at syslang.net
Some of the commands take time to run and they write to stdout and stderr as a
normal part of their operation so that we can see progress happening.
I can't seem to get the output from the remote commands (which is input to me)
to be line buffered. If I run the commands using ssh, they line buffer nicely.
If I run them through paramiko, they end up fully buffered.
I stole this code I found as a base. The code that I got looked like the
execute_capture method (below). I added the execute method so that I could
poll for the result from both stderr and stdout. Note that I am calling
channel.get_pty, but that doesn't change the fact that the results are not
line buffered.
Can anyone suggest a way to solve this?
The code I'm using follows:
#! /usr/bin/python
"""
Friendly Python SSH2 interface using paramiko
"""
import os
import sys
import tempfile
import paramiko
import select
from collections import namedtuple
ExecStatus = namedtuple('ExecStatus', 'status stdout stderr')
class Connection(object):
"""
Connects and logs into the specified hostname.
Arguments that are not given are guessed from the environment.
"""
def __init__(self,
host,
username = None,
private_key = None,
password = None,
port = 22,
blocking_cmds = True,
verbose = False,
):
self._sftp_live = False
self._sftp = None
if not username:
username = os.environ['LOGNAME']
# Log to a temporary file if requested.
if verbose:
self.templog = tempfile.mkstemp('.txt', 'ssh-')[1]
paramiko.util.log_to_file(self.templog)
else:
self.templog = False
# Begin the SSH transport.
self._transport = paramiko.Transport((host, port))
self._tranport_live = True
# Authenticate the transport.
if password:
# Using Password.
self._transport.connect(username = username, password = password)
else:
# Use Private Key.
if not private_key:
# Try to use default key.
if os.path.exists(os.path.expanduser('~/.ssh/id_rsa')):
private_key = '~/.ssh/id_rsa'
elif os.path.exists(os.path.expanduser('~/.ssh/id_dsa')):
private_key = '~/.ssh/id_dsa'
else:
raise TypeError, "You have not specified a password or key."
private_key_file = os.path.expanduser(private_key)
rsa_key = paramiko.RSAKey.from_private_key_file(private_key_file)
self._transport.connect(username = username, pkey = rsa_key)
def _sftp_connect(self):
"""Establish the SFTP connection."""
if not self._sftp_live:
self._sftp = paramiko.SFTPClient.from_transport(self._transport)
self._sftp_live = True
def get(self, remotepath, localpath = None):
"""Copies a file between the remote host and the local host."""
if not localpath:
localpath = os.path.split(remotepath)[1]
self._sftp_connect()
self._sftp.get(remotepath, localpath)
def put(self, localpath, remotepath = None):
"""Copies a file between the local host and the remote host."""
if not remotepath:
remotepath = os.path.split(localpath)[1]
self._sftp_connect()
self._sftp.put(localpath, remotepath)
def execute(self, command):
"""
Execute the given commands on a remote machine.
Return value is exit status of the remote command.
"""
# This execute method is similar to execute_capture below. The
# difference is that this method gets the stdout and stderr of
# the runnning command and forwards it on to the correct
# channel within this process.
# To do this, we use the poll(2) system call which comes from
# the select package.
def _write(fd, chan, syschan):
"""
_write internal method to check an fd from the list of fds
for a POLLIN event, read the data that's there, and if there's
anything there, then write it to the correct channel.
Return True if there was something to read.
"""
ret = False
if fd[1] & select.POLLIN:
if fd[0] == chan.channel.fileno():
ss = chan.readline()
ret = len(ss) != 0
if ret:
# No need to strip and then print with a newline.
# because every line is newline terminated.
print >> syschan, ss[:-1]
return ret
# Open a channel of type session. Same as open_channel('session')
channel = self._transport.open_session()
# Calling get_pty does get us a pty
# If the 2nd arg is 1, then it should be line buffered. Apparently,
# in this context, line buffering refers to output to the process,
# not input from the process.
channel.get_pty()
# Run command on the session of type channel. This returns immediately.
channel.exec_command(command)
# Get the stdout and stderr file descriptors. In this context, makefile
# has nothing to do with the make utility. It's about making
# file descriptors.
stdout = channel.makefile('rb', 1)
stderr = channel.makefile_stderr('rb', 1)
# Create a polling object. We really only care about POLLIN events.
po = select.poll()
po.register(stdout.channel, select.POLLIN)
po.register(stderr.channel, select.POLLIN)
# Set found_output tto True to start the loop. Set it to False
# as an initial value inside the loop, and OR the value on if any data
# got written by either channel.
found_output = True
while found_output == True:
found_output = False
fds = po.poll()
for fd in fds:
found_output |= _write(fd, stdout, sys.stdout)
found_output |= _write(fd, stderr, sys.stderr)
status = channel.recv_exit_status()
channel.close()
return status
def execute_capture(self, command):
"""
Execute the given commands on a remote machine.
Return value is a 3-tuple: exit status, stdout and stderr
Output is not written out.
"""
# Open a channel of type session. Same as open_channel('session')
channel = self._transport.open_session()
# Run command on the session of type channel. This returns immediately.
channel.exec_command(command)
# Collect stdout and stderr into lists.
# The alternative would be to harvest the 3-tuple std{in,out,err}
# of file descriptors returned by exec_command.
stdout = channel.makefile('rb', -1).readlines()
stderr = channel.makefile_stderr('rb', -1).readlines()
# Not well documented, but recv_exit_status will block until
# the command completes. Return value is the exit status.
status = channel.recv_exit_status()
channel.close()
return ExecStatus(status, stdout, stderr)
def close_transport(self):
# Close the SSH Transport.
if self._tranport_live:
self._transport.close()
self._tranport_live = False
def close_sftp(self):
# Close SFTP Connection.
if self._sftp_live:
self._sftp.close()
self._sftp_live = False
def close(self):
"""Closes the connection and cleans up."""
self.close_transport()
self.close_sftp()
# start the ball rolling.
if __name__ == "__main__":
def proc_exec(cmd):
status = myssh.execute(cmd)
print "Just ran %s with exit status %d"%(cmd, status)
# Another choice is to call
# def proc_exec(cmd):
# run = myssh.execute_capture(cmd)
# print "Just ran %s with exit status %d"%(cmd, status)
# if run.stdout:
# print "output:\n", ''.join(run.stdout)[:-1]
# if run.stderr:
# print "error:\n", ''.join(run.stderr)[:-1]
# Little test when called directly.
# Set these to your own details.
myssh = Connection('host', username='user')
proc_exec('pwd')
proc_exec('false')
proc_exec('xxx.sh')
proc_exec('cd sorr; pwd')
proc_exec('pwd')
proc_exec('tty')
myssh.close()
--
Time flies like the wind. Fruit flies like a banana. Stranger things have .0.
happened but none stranger than this. Does your driver's license say Organ ..0
Donor?Black holes are where God divided by zero. Listen to me! We are all- 000
individuals! What if this weren't a hypothetical question?
steveo at syslang.net