Directory transfers with Paramiko

Question

How do you use Paramiko to transfer complete directories? I'm trying to use:

sftp.put("/Folder1","/Folder2")

which is giving me this error -

Error : [Errno 21] Is a directory

Answer 1

You can subclass paramiko.SFTPClient and add the following method to it:

import paramiko
import os

class MySFTPClient(paramiko.SFTPClient):
    def put_dir(self, source, target):
        ''' Uploads the contents of the source directory to the target path. The
            target directory needs to exists. All subdirectories in source are 
            created under target.
        '''
        for item in os.listdir(source):
            if os.path.isfile(os.path.join(source, item)):
                self.put(os.path.join(source, item), '%s/%s' % (target, item))
            else:
                self.mkdir('%s/%s' % (target, item), ignore_existing=True)
                self.put_dir(os.path.join(source, item), '%s/%s' % (target, item))

    def mkdir(self, path, mode=511, ignore_existing=False):
        ''' Augments mkdir by adding an option to not fail if the folder exists  '''
        try:
            super(MySFTPClient, self).mkdir(path, mode)
        except IOError:
            if ignore_existing:
                pass
            else:
                raise

To use it:

transport = paramiko.Transport((HOST, PORT))
transport.connect(username=USERNAME, password=PASSWORD)
sftp = MySFTPClient.from_transport(transport)
sftp.mkdir(target_path, ignore_existing=True)
sftp.put_dir(source_path, target_path)
sftp.close()

Answer 2

You'll need to do this just like you would locally with python (if you weren't using shutils).

Combine os.walk(), with sftp.mkdir() and sftp.put(). You may also want to check each file and directory with os.path.islink() depending on whether you want to resolve symlinks or not.

Answer 3

Here's my piece of code:

import errno
import os
import stat

def download_files(sftp_client, remote_dir, local_dir):
    if not exists_remote(sftp_client, remote_dir):
        return

    if not os.path.exists(local_dir):
        os.mkdir(local_dir)

    for filename in sftp_client.listdir(remote_dir):
        if stat.S_ISDIR(sftp_client.stat(remote_dir + filename).st_mode):
            # uses '/' path delimiter for remote server
            download_files(sftp_client, remote_dir + filename + '/', os.path.join(local_dir, filename))
        else:
            if not os.path.isfile(os.path.join(local_dir, filename)):
                sftp_client.get(remote_dir + filename, os.path.join(local_dir, filename))


def exists_remote(sftp_client, path):
    try:
        sftp_client.stat(path)
    except IOError, e:
        if e.errno == errno.ENOENT:
            return False
        raise
    else:
        return True

Answer 4

This can all be done quite easily using just paramiko.

A high level summary of the code below is:
- connect to the SFTP (steps 1 to 3)
- specify your source and target folders. (step 4)
- copy them over one by one to wherever you like (I've sent them to /tmp/). (step 5)

import paramiko

# 1 - Open a transport
host="your-host-name"
port = port_number
transport = paramiko.Transport((host, port))

# 2 - Auth
password="sftp_password"
username="sftp_username"
transport.connect(username = username, password = password)

# 3 - Go!

sftp = paramiko.SFTPClient.from_transport(transport)

# 4 - Specify your source and target folders.
source_folder="some/folder/path/on/sftp"
inbound_files=sftp.listdir(source_folder)

# 5 - Download all files from that path
for file in inbound_files :
    filepath = source_folder+file
    localpath = "/tmp/"+file
    sftp.get(filepath, localpath)

Answer 5

Works for me doing something like this, all folder and files are copied to the remote server.

parent = os.path.expanduser("~")
for dirpath, dirnames, filenames in os.walk(parent):
    remote_path = os.path.join(remote_location, dirpath[len(parent)+1:])
        try:
            ftp.listdir(remote_path)
        except IOError:
            ftp.mkdir(remote_path)

        for filename in filenames:
            ftp.put(os.path.join(dirpath, filename), os.path.join(remote_path, filename))

Answer 6

You might replace sftp = self.client.open_sftp() with paramiko's one and get rid of libcloud here.

import os.path
from stat import S_ISDIR
from libcloud.compute.ssh import SSHClient
from paramiko.sftp import SFTPError

class CloudSSHClient(SSHClient):


    @staticmethod
    def normalize_dirpath(dirpath):
        while dirpath.endswith("/"):
            dirpath = dirpath[:-1]
        return dirpath


    def mkdir(self, sftp, remotepath, mode=0777, intermediate=False):
        remotepath = self.normalize_dirpath(remotepath)
        if intermediate:
            try:
                sftp.mkdir(remotepath, mode=mode)
            except IOError, e:
                self.mkdir(sftp, remotepath.rsplit("/", 1)[0], mode=mode,
                           intermediate=True)
                return sftp.mkdir(remotepath, mode=mode)
        else:
            sftp.mkdir(remotepath, mode=mode)


    def put_dir_recursively(self,  localpath, remotepath, preserve_perm=True):
        "upload local directory to remote recursively"

        assert remotepath.startswith("/"), "%s must be absolute path" % remotepath

        # normalize
        localpath = self.normalize_dirpath(localpath)
        remotepath = self.normalize_dirpath(remotepath)

        sftp = self.client.open_sftp()

        try:
            sftp.chdir(remotepath)
            localsuffix = localpath.rsplit("/", 1)[1]
            remotesuffix = remotepath.rsplit("/", 1)[1]
            if localsuffix != remotesuffix:
                remotepath = os.path.join(remotepath, localsuffix)
        except IOError, e:
            pass

        for root, dirs, fls in os.walk(localpath):
            prefix = os.path.commonprefix([localpath, root])
            suffix = root.split(prefix, 1)[1]
            if suffix.startswith("/"):
                suffix = suffix[1:]

            remroot = os.path.join(remotepath, suffix)

            try:
                sftp.chdir(remroot)
            except IOError, e:
                if preserve_perm:
                    mode = os.stat(root).st_mode & 0777
                else:
                    mode = 0777
                self.mkdir(sftp, remroot, mode=mode, intermediate=True)
                sftp.chdir(remroot)

            for f in fls:
                remfile = os.path.join(remroot, f)
                localfile = os.path.join(root, f)
                sftp.put(localfile, remfile)
                if preserve_perm:
                    sftp.chmod(remfile, os.stat(localfile).st_mode & 0777)

Answer 7

I don't think you can do that. Look up the documentation for os.walk and copy each file "manually".

Answer 8

This is my first StackOverflow answer. I had a task today which is similar to this. So, I tried to find a direct way to copy entire folder from windows to linux using python and paramiko. After a little research, I came up with this solution which works for smaller size folders with subfolders and files in it.

This solution first makes the zip file for the current folder (os.walk() is very much helpful here), then copies to destination server and unzip there.

zipHere = zipfile.ZipFile("file_to_copy.zip", "w")

for root, folders, files in os.walk(FILE_TO_COPY_PATH):
    for file in files:
        zipHere.write(os.path.join(root, file), arcname=os.path.join(os.path.relpath(root, os.path.dirname(FILE_TO_COPY_PATH)), file))
    for folder in folders:
        zipHere.write(os.path.join(root, folder), arcname=os.path.join(os.path.relpath(root, os.path.dirname(FILE_TO_COPY_PATH)), folder))
zipHere.close()

# sftp is the paramiko.SFTPClient connection
sftp.put('local_zip_file_location','remote_zip_file_location')

# telnet_conn is the telnetlib.Telnet connection
telnet_conn.write('cd cd_to_zip_file_location')
telnet_conn.write('unzip -o file_to_copy.zip')

Answer 9

Paramiko does not support directory transfers on its own. You have to implement it, as many existing answers here show.

You can check pysftp code. It's a wrapper around Paramiko that supports recursive operations. See

• pysftp.Connection.put_r()
• pysftp.Connection.get_r()

I do not recommend using pysftp code directly (pysftp vs. Paramiko) though, as it is not maintained anymore and has some bugs. For working standalone portable Paramiko-only code based on pysftp see my answers to:

• Python pysftp get_r from Linux works fine on Linux but not on Windows
• Python pysftp put_r does not work on Windows

Answer 10

my answer is similar with above just make a list, and then transfer one by one.

import paramiko

ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname='11.11.11.1111', username='root', password='********', port=22)
sftp_client = ssh.open_sftp()
source_folder = '/var/ftp/file_pass'
local_folder = 'C:/temp/file_pass'
inbound_files = sftp_client.listdir(source_folder)
print(inbound_files)

for ele in inbound_files:
    try:
        path_from = source_folder + '/' + ele
        path_to = local_folder + '/'+ ele
        sftp_client.get(path_from, path_to)
    except:
        print(ele)

sftp_client.close()
ssh.close()

Answer 11

This is my approach but and the code handle hidden files also

import paramiko

ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect("localhost", port=19000, username="test", password="test")
sftp = ssh.open_sftp()

source_folder="/etc/"
inbound_files=sftp.listdir(source_folder)


for file in inbound_files :
    filepath = source_folder+file
    localpath = "/home/"+file
    # check hidden files
    if file.startswith('.'):
        pass
    else:
        sftp.get(filepath, localpath)
    

Answer 12

As far as I know, Paramiko does not support recursive file upload. However, I have found a solution for recursive upload using Paramiko here. Follows an excerpt of their recursive upload function:

   def _send_recursive(self, files):
        for base in files:
            lastdir = base
            for root, dirs, fls in os.walk(base):
                # pop back out to the next dir in the walk
                while lastdir != os.path.commonprefix([lastdir, root]):
                    self._send_popd()
                    lastdir = os.path.split(lastdir)[0]
                self._send_pushd(root)
                lastdir = root
                self._send_files([os.path.join(root, f) for f in fls])

You may try to either use their function SCPClient.put invoking the above function for recursive upload or implement it on your own.

Answer 13

If you would like to have parallel copy per folder you can use (keep in mind that it will ignore files that already exists localy):

def get_folders(sftp_client, remote_dir, local_dir):
    if not exists_remote(sftp_client, remote_dir):
        return

    if not os.path.exists(local_dir):
        os.mkdir(local_dir)

    for filename in sftp_client.listdir(remote_dir):
        remote_dir_path = f"{remote_dir}/{filename}"
        print(f"downloading {remote_dir_path}")
        current_stat = sftp_client.stat(remote_dir_path)
        if stat.S_ISDIR(current_stat.st_mode):
            get_folders(sftp_client, remote_dir_path, os.path.join(local_dir, filename))
        else:
            if not os.path.isfile(os.path.join(local_dir, filename)):
                sftp_client.get(remote_dir_path, os.path.join(local_dir, filename))


def exists_remote(sftp_client, path):
    try:
        sftp_client.stat(path)
    except IOError as e:
        if e.errno == errno.ENOENT:
            return False
        raise
    else:
        return True


def copy_from_server(dir_path):
    import paramiko

    server = "A"
    username = "B"
    password = "C"
    remote_path = ""
    local_path = ""

    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    ssh.connect(server, username=username, password=password)
    sftp = ssh.open_sftp()
    print("connected")
    get_folders(sftp, f"{remote_path}/{dir_path}",
                   rf"{local_path}\{dir_path}")
    print("downloaded")
    sftp.close()
    ssh.close()


def parallel_copy_from_server():
    dir_names = ['A', 'B']
    NUM_OF_CPUS = multiprocessing.cpu_count()
    with Pool(NUM_OF_CPUS) as p:
        results_list = p.map(copy_from_server, dir_names)

Answer 14

Based on the top voted answer by skoll, I made a solution that may be more high-level (recursive is an option; has return values; if <local_path> is a file <remote_path> can be a directory or filename) and modern (type hints; pathlib):

from pathlib import Path

import paramiko
from paramiko.common import o777


def mkdir(client: paramiko.SFTP, path: Path | str, mode: int = o777, exists_ok: bool = False) -> None:
    """Augments mkdir by adding an option to not fail if the folder exists"""
    try:
        client.mkdir(str(path), mode)
    except IOError as err:
        if exists_ok:
            pass
        else:
            raise err


def upload(
    client: paramiko.SFTP, local_path: Path | str, remote_path: Path | str, recursive: bool = False
) -> paramiko.SFTPAttributes | list[paramiko.SFTPAttributes]:
    """
    Upload file (or directory if <recursive> is True) to a remote path.

    The correct uploading of files is verified by examining their sizes.
    """

    local_path = Path(local_path)
    remote_path = Path(remote_path)

    if recursive and local_path.is_dir():
        mkdir(client, remote_path / local_path.name, exists_ok=True)
        return [upload(client, sub_path, remote_path / local_path.name, recursive) for sub_path in local_path.iterdir()]
    else:
        try:
            return client.put(str(local_path), str(remote_path))
        except OSError as err:
            if str(err) == "Specified file is a directory." and local_path.is_file():
                return client.put(str(local_path), str(remote_path / local_path.name))
            raise err