Skip to content

Instantly share code, notes, and snippets.

@omaraflak
Last active February 11, 2021 15:43
Show Gist options
  • Save omaraflak/8e6a127f659af796bf72a3448c9fd88e to your computer and use it in GitHub Desktop.
Save omaraflak/8e6a127f659af796bf72a3448c9fd88e to your computer and use it in GitHub Desktop.
Fast way to send arbitrarily large files using socket.io. The code will send the file chunks by chunks and never load the whole file into memory.
const client = require('socket.io-client')('http://localhost:5000')
const siof = require('./socket.io-file')
client.on('connect', () => {
console.log('connected to server')
client.once('ready', () => {
const metadata = { filename: 'file2.pdf' } // filename on the receiving end
siof(client).emit('file', 'filepath/to/file.pdf', metadata, (sent, total) => {
const percentage = (100 * sent / total).toFixed(2)
console.log(`${percentage}%`)
}, () => {
console.log('file sent')
client.close()
}, () => {
console.log('transfer canceled')
})
})
})
const server = require('socket.io')(5000)
const siof = require('./socket.io-file')
server.on('connection', socket => {
console.log('client connected')
siof(socket).on('file', './output', (err, data) => {
console.log(data)
})
socket.emit('ready')
})
console.log('Listening at http://localhost:5000')
const fs = require('fs')
const path = require('path')
const { nanoid } = require('nanoid')
const CHUNK_SIZE = 1000000
const createDirIfNotExistsRecursive = directory => {
if (!fs.existsSync(directory)) {
fs.mkdirSync(directory, { recursive: true })
}
}
const receiveFile = (socket, key, destination, callback) => {
createDirIfNotExistsRecursive(destination)
const fid = nanoid(10)
const tmp = path.join(destination, fid)
const stream = fs.createWriteStream(tmp)
socket.on(key, part => {
if (part.metadata) {
stream.close()
fs.renameSync(tmp, path.join(destination, part.metadata.filename))
socket.emit(`${key}:${part.id}:end`)
callback(null, part.metadata)
} else {
stream.write(part.chunk)
socket.emit(`${key}:${part.id}:next`)
}
})
}
const sendFile = (socket, key, filepath, metadata, onProgress, onDone, onCanceled) => {
const id = nanoid(5)
const onNextKey = `${key}:${id}:next`
const onAbortKey = `${key}:${id}:abort`
const onEndKey = `${key}:${id}:end`
const buffer = new Buffer(CHUNK_SIZE)
const filesize = fs.statSync(filepath).size
const fd = fs.openSync(filepath)
let bytesRead = 0
let bytesSent = 0
const removeAllListeners = () => {
socket.removeAllListeners(onNextKey)
socket.removeAllListeners(onAbortKey)
socket.removeAllListeners(onEndKey)
}
const sendNextChunk = () => {
bytesRead = fs.readSync(fd, buffer, 0, CHUNK_SIZE, null)
bytesSent += bytesRead
if (bytesRead > 0) {
socket.emit(key, { id, chunk: buffer.slice(0, bytesRead) })
onProgress(bytesSent, filesize)
} else {
socket.emit(key, { id, metadata })
fs.closeSync(fd)
}
}
const abortTransfer = () => {
removeAllListeners()
fs.closeSync(fd)
onCanceled()
}
const endTransfer = () => {
removeAllListeners()
onDone()
}
socket.on(onNextKey, sendNextChunk)
socket.on(onAbortKey, abortTransfer)
socket.on(onEndKey, endTransfer)
sendNextChunk()
}
const siof = socket => {
return {
on: (key, destination, callback) => {
receiveFile(socket, key, destination, callback)
},
emit: (key, filepath, metadata, onProgress, onDone, onCanceled) => {
sendFile(socket, key, filepath, metadata, onProgress, onDone, onCanceled)
}
}
}
module.exports = siof
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment