push_io.py

pythonCopy code
import paramiko

def stream_data_from_sftp(hostname, port, username, password, remote_path, chunk_size=1024):
    transport = paramiko.Transport((hostname, port))
    transport.connect(username=username, password=password)
    sftp = paramiko.SFTPClient.from_transport(transport)

    with sftp.open(remote_path, 'rb') as file:
        while chunk := file.read(chunk_size):
            yield chunk

    sftp.close()
    transport.close()

# Example usage
hostname = 'sftp.example.com'
port = 22
username = 'user'
password = 'pass'
remote_path = '/path/to/large_file'
for chunk in stream_data_from_sftp(hostname, port, username, password, remote_path):
    # Process each chunk
    print(chunk)