Skip to content

Instantly share code, notes, and snippets.

@g-pechorin
Created May 16, 2021 15:12
Show Gist options
  • Save g-pechorin/2107b29e7b6b59df173a149eb625bd99 to your computer and use it in GitHub Desktop.
Save g-pechorin/2107b29e7b6b59df173a149eb625bd99 to your computer and use it in GitHub Desktop.
crude way to embed python in Scala. uses a bunch of implicit classes that should be obvious
package peterlavalle.pybx
import java.io.{File, OutputStream}
import java.util.Random
import peterlavalle._
import scala.sys.process.ProcessLogger
/**
* provides a way to run a sandboxed python
*
* my bitter attempt to normalise python execution.
* this time i'm producing a wrapper in Scala-land with the intent to do os specific implementations.
*/
trait PyBed {
/**
* the "real" version that the implementation must provide
*/
def open(
// folder to work in
dir: File,
// the command to run with python-c given the port number
run: Int => String,
// pip packages to install
pip: Seq[String],
// source files to write
src: Int => Map[String, String],
// where to send the stdout and stderr
log: ProcessLogger = ProcessLogger(System.out.println, System.err.println),
// where to send whatever that is written to the socket
out: OutputStream,
// should we create any threads as a daemon
isDaemon: Boolean = true,
// create a random port
portGen: () => Int = () => new Random().nextInt(2020) + 1024
): OutputStream
/**
* the laziest
*/
def open(
// the script to run - will be named after the call-stack-hash
src: Int => String,
// pip packages to install
pip: Seq[String],
// where to send whatever that is written to the socket
out: OutputStream,
): OutputStream = open(null, src, pip, out)
/**
* the "lazy" version that I'm probably going to end up using
*/
def open(
// folder to work in
dir: File,
// the script to run - will be named after the call-stack-hash
src: Int => String,
// pip packages to install
pip: Seq[String],
// where to send whatever that is written to the socket
out: OutputStream,
): OutputStream = {
val md5: String = {
// the trace, minus "us"
val trace: List[StackTraceElement] =
Thread.currentThread()
.getStackTrace
.toList.tail
.dropWhile(_.getClassName.endsWith(".PyBed"))
// simple name to make life easier
val name: String =
trace
.head.getClassName
.reverse.dropWhile('$' == _).takeWhile('.' != _).reverse
// we can do the/a "to string" because it's a list ... but i choose safety
val hash =
trace
.foldLeft("")((_: String) + (_: StackTraceElement))
.md5
// computed value
name + "-" + hash
}
open(
if (null != dir)
dir
else
(dir / "target" / md5).EnsureMkDirs,
port => s"$md5.py",
pip,
port => Map(s"$md5.py" -> src(port)),
ProcessLogger(System.out.println, System.err.println),
out
)
}
}
package peterlavalle.pybx
import java.io.{File, FileWriter, OutputStream}
import java.net.{ServerSocket, Socket}
import peterlavalle._
import scala.sys.process.{Process, ProcessLogger}
/**
* for my workstation (which was named diagon for some reason)
*/
object PyBedDiagonal1 extends PyBed {
/**
* the "real" version that the implementation must provide
*/
override def open(
dir: File,
cmd: Int => String,
pip: Seq[String],
src: Int => Map[String, String],
log: ProcessLogger,
out: OutputStream,
isDaemon: Boolean,
portGen: () => Int
): OutputStream = {
// choose a port
val port = portGen()
// create the TCP server
val server = new ServerSocket(port)
// create the python process as/in thread
val process: Thread =
new Thread() {
override def run(): Unit = {
log.out("dir = " + dir.AbsolutePath)
// setup the env with pip
log.out("venv = " + {
Process(
command = "python -m venv ./",
cwd = dir.EnsureMkDirs
) ! log
})
// update out pip
log.out("pip = " + {
Process(
command = "CMD /C \"Scripts\\activate.bat && pip install pip --upgrade\"",
cwd = dir.EnsureMkDirs
) ! log
})
// check the pip details (because it keeps crashing)
log.out("pip -V = " + {
Process(
command = "CMD /C \"Scripts\\activate.bat && pip -V\"",
cwd = dir.EnsureMkDirs
) ! log
})
// install our packages
pip.foreach {
pip =>
log.out(s"pip $pip = " + {
Process(
command = "CMD /C \"Scripts\\activate.bat && pip install " + pip + "\"",
cwd = dir.EnsureMkDirs
) ! log
})
}
// write our scripts
src(port).foreach {
case (name, code) =>
new FileWriter(dir / name)
.append(code)
.close()
log.out("wrote " + name)
}
// run the script
log.out("run = " + {
Process(
command = "CMD /C \"Scripts\\activate.bat && python " + (cmd(port)) + "\"",
cwd = dir.EnsureMkDirs
) ! log
})
}
setDaemon(isDaemon)
start()
}
// accept a connection
val socket: Socket = {
TODO("something magical to handle crashed python")
server.accept()
}
// grab
val output = socket.getOutputStream
val input = socket.getInputStream
// launch the pump-pipe thread
val pump: Thread =
new Thread() {
override def run(): Unit = {
val buffer = Array.ofDim[Byte](32)
while (true) {
val read = input.read(buffer)
println(read)
out.write(buffer, 0, read)
}
}
setDaemon(isDaemon)
start()
}
new OutputStream {
override def write(i: Int): Unit = output.write(i)
override def write(b: Array[Byte]): Unit = output.write(b)
override def write(b: Array[Byte], off: Int, len: Int): Unit = output.write(b, off, len)
override def flush(): Unit = output.flush()
override def close(): Unit = {
output.close()
// close the output
output.close()
// close the socket
socket.close()
// await the thread(s)
process.join()
pump.join()
// close the server
server.close()
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment