Created
May 16, 2021 15:12
-
-
Save g-pechorin/2107b29e7b6b59df173a149eb625bd99 to your computer and use it in GitHub Desktop.
crude way to embed python in Scala. uses a bunch of implicit classes that should be obvious
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package peterlavalle.pybx | |
import java.io.{File, OutputStream} | |
import java.util.Random | |
import peterlavalle._ | |
import scala.sys.process.ProcessLogger | |
/** | |
* provides a way to run a sandboxed python | |
* | |
* my bitter attempt to normalise python execution. | |
* this time i'm producing a wrapper in Scala-land with the intent to do os specific implementations. | |
*/ | |
trait PyBed { | |
/** | |
* the "real" version that the implementation must provide | |
*/ | |
def open( | |
// folder to work in | |
dir: File, | |
// the command to run with python-c given the port number | |
run: Int => String, | |
// pip packages to install | |
pip: Seq[String], | |
// source files to write | |
src: Int => Map[String, String], | |
// where to send the stdout and stderr | |
log: ProcessLogger = ProcessLogger(System.out.println, System.err.println), | |
// where to send whatever that is written to the socket | |
out: OutputStream, | |
// should we create any threads as a daemon | |
isDaemon: Boolean = true, | |
// create a random port | |
portGen: () => Int = () => new Random().nextInt(2020) + 1024 | |
): OutputStream | |
/** | |
* the laziest | |
*/ | |
def open( | |
// the script to run - will be named after the call-stack-hash | |
src: Int => String, | |
// pip packages to install | |
pip: Seq[String], | |
// where to send whatever that is written to the socket | |
out: OutputStream, | |
): OutputStream = open(null, src, pip, out) | |
/** | |
* the "lazy" version that I'm probably going to end up using | |
*/ | |
def open( | |
// folder to work in | |
dir: File, | |
// the script to run - will be named after the call-stack-hash | |
src: Int => String, | |
// pip packages to install | |
pip: Seq[String], | |
// where to send whatever that is written to the socket | |
out: OutputStream, | |
): OutputStream = { | |
val md5: String = { | |
// the trace, minus "us" | |
val trace: List[StackTraceElement] = | |
Thread.currentThread() | |
.getStackTrace | |
.toList.tail | |
.dropWhile(_.getClassName.endsWith(".PyBed")) | |
// simple name to make life easier | |
val name: String = | |
trace | |
.head.getClassName | |
.reverse.dropWhile('$' == _).takeWhile('.' != _).reverse | |
// we can do the/a "to string" because it's a list ... but i choose safety | |
val hash = | |
trace | |
.foldLeft("")((_: String) + (_: StackTraceElement)) | |
.md5 | |
// computed value | |
name + "-" + hash | |
} | |
open( | |
if (null != dir) | |
dir | |
else | |
(dir / "target" / md5).EnsureMkDirs, | |
port => s"$md5.py", | |
pip, | |
port => Map(s"$md5.py" -> src(port)), | |
ProcessLogger(System.out.println, System.err.println), | |
out | |
) | |
} | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package peterlavalle.pybx | |
import java.io.{File, FileWriter, OutputStream} | |
import java.net.{ServerSocket, Socket} | |
import peterlavalle._ | |
import scala.sys.process.{Process, ProcessLogger} | |
/** | |
* for my workstation (which was named diagon for some reason) | |
*/ | |
object PyBedDiagonal1 extends PyBed { | |
/** | |
* the "real" version that the implementation must provide | |
*/ | |
override def open( | |
dir: File, | |
cmd: Int => String, | |
pip: Seq[String], | |
src: Int => Map[String, String], | |
log: ProcessLogger, | |
out: OutputStream, | |
isDaemon: Boolean, | |
portGen: () => Int | |
): OutputStream = { | |
// choose a port | |
val port = portGen() | |
// create the TCP server | |
val server = new ServerSocket(port) | |
// create the python process as/in thread | |
val process: Thread = | |
new Thread() { | |
override def run(): Unit = { | |
log.out("dir = " + dir.AbsolutePath) | |
// setup the env with pip | |
log.out("venv = " + { | |
Process( | |
command = "python -m venv ./", | |
cwd = dir.EnsureMkDirs | |
) ! log | |
}) | |
// update out pip | |
log.out("pip = " + { | |
Process( | |
command = "CMD /C \"Scripts\\activate.bat && pip install pip --upgrade\"", | |
cwd = dir.EnsureMkDirs | |
) ! log | |
}) | |
// check the pip details (because it keeps crashing) | |
log.out("pip -V = " + { | |
Process( | |
command = "CMD /C \"Scripts\\activate.bat && pip -V\"", | |
cwd = dir.EnsureMkDirs | |
) ! log | |
}) | |
// install our packages | |
pip.foreach { | |
pip => | |
log.out(s"pip $pip = " + { | |
Process( | |
command = "CMD /C \"Scripts\\activate.bat && pip install " + pip + "\"", | |
cwd = dir.EnsureMkDirs | |
) ! log | |
}) | |
} | |
// write our scripts | |
src(port).foreach { | |
case (name, code) => | |
new FileWriter(dir / name) | |
.append(code) | |
.close() | |
log.out("wrote " + name) | |
} | |
// run the script | |
log.out("run = " + { | |
Process( | |
command = "CMD /C \"Scripts\\activate.bat && python " + (cmd(port)) + "\"", | |
cwd = dir.EnsureMkDirs | |
) ! log | |
}) | |
} | |
setDaemon(isDaemon) | |
start() | |
} | |
// accept a connection | |
val socket: Socket = { | |
TODO("something magical to handle crashed python") | |
server.accept() | |
} | |
// grab | |
val output = socket.getOutputStream | |
val input = socket.getInputStream | |
// launch the pump-pipe thread | |
val pump: Thread = | |
new Thread() { | |
override def run(): Unit = { | |
val buffer = Array.ofDim[Byte](32) | |
while (true) { | |
val read = input.read(buffer) | |
println(read) | |
out.write(buffer, 0, read) | |
} | |
} | |
setDaemon(isDaemon) | |
start() | |
} | |
new OutputStream { | |
override def write(i: Int): Unit = output.write(i) | |
override def write(b: Array[Byte]): Unit = output.write(b) | |
override def write(b: Array[Byte], off: Int, len: Int): Unit = output.write(b, off, len) | |
override def flush(): Unit = output.flush() | |
override def close(): Unit = { | |
output.close() | |
// close the output | |
output.close() | |
// close the socket | |
socket.close() | |
// await the thread(s) | |
process.join() | |
pump.join() | |
// close the server | |
server.close() | |
} | |
} | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment