Pull and publish .NetCore applications from command line

couchpotatoserver.py

from flask import Flask
from flask import request, Response
from flask_cors import CORS, cross_origin
from crontab import CronTab
import os
import psutil
import subprocess
import select
import json
from flask import jsonify
import uuid
import glob

# Cron docs https://stackabuse.com/scheduling-jobs-with-python-crontab/

class CronJobItem(dict):
    def __init__(self):
        dict.__init__(self)
    comment = ""
    command = ""
    hour = 0
    minute = 0

app = Flask(__name__)
CORS(app)
app.config['CORS_HEADERS'] = 'Content-Type'
app.debug = False
isUpdating = False
app.config['SECRET_KEY'] = 'secret!'


def ifProcessRunning(processName):
    for proc in psutil.process_iter():
        try:
            if processName.lower() in proc.name().lower():
                return True
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
            pass
    return False

def getLastRunLog():
    data = {}
    with open("/couchpotato/logs/lastrun.log") as outfile:
        data = json.load(outfile)

    return data

def getLogFile(log):
    with open("/couchpotato/logs/" + log + ".log") as outfile:
        return outfile.read()

def getAppSettings():
    data = {}
    with open("/couchpotato/application/appsettings.json") as outfile:
        data = json.load(outfile)

    return data

def getPluginPath():
    appsettings=getAppSettings()
    pluginPath=appsettings["pluginPath"]

    return pluginPath

def getPluginSettings():
    pluginPath=getPluginPath()
    path=pluginPath + "/config.json"
    return getJsonFile(path)
    

def setPluginSettings(pluginId, settings):
    pluginPath=getPluginPath()

    data = {}
    path=pluginPath + "/config.json"
    print(path)
    with open(path) as outfile:
        data = json.load(outfile)

    data[pluginId] = settings

    with open(path, 'w') as writeFile:
        json.dump(data, writeFile)

    return data

def getJsonFile(path):
    data = ''
    with open(path) as outfile:
        data = json.load(outfile)

    return data

def getInstalledPlugins():
    arr = []
    path=getPluginPath()
    files = sorted(os.listdir(path))
    for filename in files:
        if filename.endswith(".dll") or filename.endswith("_inactive"):
            assemblyPath = path + '/' + filename
            version = getAssemblyVersion(assemblyPath)
            isActive = not assemblyPath.endswith('_inactive')
            arr.append({'name': os.path.splitext(filename)[0], 'version':str(version,'UTF-8').strip(), 'active': isActive })

    return jsonify(arr)

def getAssemblyVersion(path):
    if os.path.exists(path):
        ps = subprocess.Popen(('strings', path), stdout=subprocess.PIPE)
        output = subprocess.check_output(('egrep', '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$'), stdin=ps.stdout)
        ps.wait()
        return output
    
    return ''

@app.route('/')
def hello():
    return "Hello, I'm Couchpotato"


@app.route('/ping/')
@cross_origin()
def ping():
    return 'pong'


@app.route('/plugins/settings/file/', methods=['GET'])
@cross_origin()
def pluginsettingsFileGet():
    path = request.args.get('path')
    if not os.path.exists(path):
        with open(path, 'w') as writeFile:
            json.dump({}, writeFile)

    with open(path, "r") as f:
        content = f.read()
    return Response(content, mimetype='text/plain')

@app.route('/plugins/settings/file/', methods=['POST'])
@cross_origin()
def pluginsettingsFileSet():
    settings = request.get_json()

    with open(settings["path"], 'w') as writeFile:
        json.dump(settings["contents"], writeFile)

    return "ok"

@app.route('/plugins/settings/<plugin>', methods=['GET'])
@cross_origin()
def pluginsettingsGet(plugin):
    settings = getPluginSettings()
    if plugin in settings:
        return jsonify(settings[plugin])
    else:
        return jsonify({})

@app.route('/plugins/settings/<plugin>', methods=['POST'])
@cross_origin()
def pluginsettingsSave(plugin):
    data = request.json

    result = setPluginSettings(plugin, data)
    return jsonify(result)

@app.route('/plugins/uninstall/<plugin>', methods=['GET'])
@cross_origin()
def uninstallPlugin(plugin):
    pluginPath=getPluginPath()
    os.remove(pluginPath + "/" + plugin + ".dll") 
    return "ok"

@app.route('/plugins/activate/<plugin>', methods=['GET'])
@cross_origin()
def activatePlugin(plugin):
    pluginPath=getPluginPath()
    oldname = pluginPath + "/" + plugin + ".dll_inactive"
    newname = pluginPath + "/" + plugin + ".dll"
    if os.path.exists(oldname):
        os.rename(oldname, newname) 
        return "True"
    return "False"

@app.route('/plugins/deactivate/<plugin>', methods=['GET'])
@cross_origin()
def deactivatePlugin(plugin):
    pluginPath=getPluginPath()
    oldname = pluginPath + "/" + plugin + ".dll"
    newname = pluginPath + "/" + plugin + ".dll_inactive"
    if os.path.exists(oldname):
        os.rename(oldname, newname) 
        return "True"
    return "False"

@app.route('/plugins/installed/')
@cross_origin()
def pluginsInstalled():
    settings = getInstalledPlugins()
    return settings

@app.route('/couchpotato/version/', methods=['GET'])
@cross_origin()
def couchpotatoVersion():
    return getAssemblyVersion('/couchpotato/application/couchpotato.dll')

@app.route('/couchpotato/lastrun/', methods=['GET'])
@cross_origin()
def couchpotatoLastRun():
    log = getLastRunLog()
    return jsonify(log)

@app.route('/couchpotato/logs/', methods=['GET'])
@cross_origin()
def couchpotatoLogs():
    logs = glob.glob("/couchpotato/logs/*.log")
    return jsonify(logs)

@app.route('/couchpotato/logs/<log>', methods=['GET'])
@cross_origin()
def couchpotatoLog(log):
    log = getLogFile(log)
    return log


@app.route('/couchpotato/logs/<log>', methods=['DELETE'])
@cross_origin()
def couchpotatoLogClear(log):
    if os.path.exists("/couchpotato/logs/"+ log + ".log"):
        os.remove("/couchpotato/logs/"+ log + ".log")
        return 'ok'

    return 'ok'

@app.route('/backend/update/', methods=['GET'])
@cross_origin()
def backendUpdate():
    subprocess.call(['wget', '-O', '/couchpotato/server/couchpotatosocketserver.py', 'https://gist.githubusercontent.com/mattiasnorell/35033284096e5e680cdc10636f7769f8/raw/couchpotatosocketserver.py'])
    subprocess.call(['wget', '-O', '/couchpotato/server/couchpotatoserver.py', 'https://gist.githubusercontent.com/mattiasnorell/35033284096e5e680cdc10636f7769f8/raw/couchpotatoserver.py'])
    
    return "ok"

@app.route('/cron/status/', methods=['GET'])
@cross_origin()
def cronStatus():
    output = subprocess.check_output(('service', 'cron', 'status'))
    return output


@app.route('/cron/', methods=['GET'])
@cross_origin()
def cronGet():
    showInactive = request.args.get('showInactive') == "true"
    cron = CronTab(user='root')
    jobs = []

    for job in cron:
        rendered=job.render().split(" ")
        if job.is_enabled() == True:
            jobs.append({"hour": rendered[1], "minute": rendered[0], "command": job.command, "id": job.comment, "active": job.is_enabled()})
        elif showInactive == True and job.is_enabled() == False:
            jobs.append({"hour": rendered[2], "minute": rendered[1], "command": job.command, "id": job.comment, "active": job.is_enabled()})

    return jsonify(jobs)

@app.route('/cron/', methods=['POST'])
@cross_origin()
def cronCreate():
    cron = CronTab(user='root')
    newJob = request.get_json()
    job = cron.new(command=newJob["command"], comment="couchpotato-" + str(uuid.uuid4()))
    job.hour.on(newJob["hour"])
    job.minute.on(newJob["minute"])
    job.enable(True)

    cron.write()
    
    return "ok"

@app.route('/cron/', methods=['PUT'])
@cross_origin()
def cronSave():
    cron = CronTab(user='root')
    jobs = request.get_json()
    
    for job in jobs:
        iter = cron.find_comment(job["id"])
        
        for iterJob in iter:
            iterJob.hour.on(job["hour"])
            iterJob.minute.on(job["minute"])
            iterJob.command = job["command"]
            iterJob.comment = job["id"]
            iterJob.enable(job["active"])

    cron.write()
    return "ok"

@app.route('/cron/<id>/', methods=['DELETE'])
@cross_origin()
def cronRemove(id):
    cron = CronTab(user='root')
    jobs = cron.find_comment(id)
    for job in jobs:
        cron.remove(job)

    cron.write()
    return "ok"

@app.route('/cron/delete/')
@cross_origin()
def cronDelete(id):
    my_cron = CronTab(user='root')
    my_cron.remove(comment=id)

@app.route('/cron/restart/')
@cross_origin()
def restartCron():
    os.system('/etc/init.d/cron restart')
    return "ok"

@app.route('/restartbackend/')
@cross_origin()
def restartApi():
    os.system('/couchpotato/server/restart-backend.sh')
    return "ok"


@app.route('/status/')
@cross_origin()
def status():
    if ifProcessRunning('couchpotato'):
        return "running"
    else:
        return 'not_running'

@app.route('/run/')
@cross_origin()
def run():

    if ifProcessRunning('couchpotato'):
        return "running"
    else:
        def inner():
            proc = subprocess.Popen(
                ['/couchpotato/run.sh'],
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )

            try:
                awaiting = [proc.stdout, proc.stderr]
                while awaiting:
                    ready, _, _ = select.select(awaiting, [], [])
                    for pipe in ready:
                        line = pipe.readline()
                        if line:
                            yield line.rstrip() + b'<br/>\n'
                        else:
                            awaiting.remove(pipe)
                if proc.poll() is None:
                    print(
                        "process closed stdout and stderr but didn't terminate; terminating now.")
                    proc.terminate()

            except GeneratorExit:
                print('client disconnected, killing process')
                proc.terminate()

            ret_code = proc.wait()
            print("process return code:", ret_code)

        return Response(inner(), mimetype='text/html')


if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8001)

couchpotatosocketserver.py

#!/usr/bin/env python
import os
import psutil
import subprocess
import asyncio
import json
import websockets
import select
from shlex import split
from subprocess import Popen, PIPE


def ifProcessRunning(processName):
    for proc in psutil.process_iter():
        try:
            if processName.lower() in proc.name().lower():
                return True
        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
            pass
    return False


async def run(websocket, path, configPath):
    proc = subprocess.Popen(
        "%s %s" % (path,configPath),
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    try:
        awaiting = [proc.stdout, proc.stderr]
        while awaiting:
            ready, _, _ = select.select(awaiting, [], [])
            for pipe in ready:
                line = pipe.readline()
                if line:
                    message = json.dumps(
                        {'data': line.rstrip().decode("utf-8")})
                    await websocket.send(message)
                else:
                    awaiting.remove(pipe)
        if proc.poll() is None:
            proc.terminate()

    except GeneratorExit:
        print('client disconnected, killing process')
        proc.terminate()

async def installPlugin(websocket, path, configPath,accessToken):
    proc = subprocess.Popen(
        "%s %s %s" % (path,configPath, accessToken),
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    try:
        awaiting = [proc.stdout, proc.stderr]
        while awaiting:
            ready, _, _ = select.select(awaiting, [], [])
            for pipe in ready:
                line = pipe.readline()
                if line:
                    message = json.dumps(
                        {'data': line.rstrip().decode("utf-8")})
                    await websocket.send(message)
                else:
                    awaiting.remove(pipe)
        if proc.poll() is None:
            proc.terminate()

    except GeneratorExit:
        print('client disconnected, killing process')
        proc.terminate()


async def reinstall(websocket, gitHubToken):
    args = ['wget', '-r', '-l', '1', '-p', '-P', '/couchpotato/server/install.sh', 'http://couchpotato.automagiskdatabehandling.se/install.sh']
    p1 = Popen(
        args, stdout=PIPE,shell=True)
    proc = Popen(
        "/couchpotato/server/install.sh %s" % (gitHubToken),
        shell=True,
        stdin=p1.stdout,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    try:
        awaiting = [proc.stdout, proc.stderr]
        while awaiting:
            ready, _, _ = select.select(awaiting, [], [])
            for pipe in ready:
                line = pipe.readline()
                if line:
                    message = json.dumps(
                        {'data': line.rstrip().decode("utf-8")})
                    await websocket.send(message)
                else:
                    awaiting.remove(pipe)
        if proc.poll() is None:
            proc.terminate()

    except GeneratorExit:
        print('client disconnected, killing process')
        proc.terminate()

async def importAction(websocket, path, configPath):
    await run(websocket, path, configPath)
    message = json.dumps({'data': 'done'})
    await websocket.send(message)

async def installPluginAction(websocket, path, configPath, accessToken):
    await installPlugin(websocket, path, configPath, accessToken)
    message = json.dumps({'data': 'done'})
    await websocket.send(message)

async def reinstallAction(websocket, gitHubToken):
    await reinstall(websocket, gitHubToken)
    message = json.dumps({'data': 'done'})
    await websocket.send(message)

async def runAction(websocket, command):
    await run(websocket, command, "")
    message = json.dumps({'data': 'done'})
    await websocket.send(message)

async def register(websocket):
    message = json.dumps({'data': 'connection_open'})
    await websocket.send(message)


async def counter(websocket, path):
    await register(websocket)
    try:
        async for message in websocket:
            data = json.loads(message)
            if data["action"] == "import":
                if ifProcessRunning('couchpotato'):
                    message = json.dumps({'data': 'running'})
                    await websocket.send(message)
                else:
                    
                    config = "http://couchpotato.automagiskdatabehandling.se/config/%s.json" % (data["url"])
                    await importAction(websocket, '/couchpotato/run.sh', config)

            if data["action"] == "reinstall":
                gitHubToken = data["accessToken"]
                await reinstallAction(websocket, gitHubToken)

            if data["action"] == "run":
                await runAction(websocket, data["url"])

            if data["action"] == "installplugin":
                await installPluginAction(websocket, '/couchpotato/server/install-plugin.sh', data["url"], data["accessToken"])

    finally:
        await websocket.close()


start_server = websockets.serve(counter, "0.0.0.0", 8000)

asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()

install-plugin.sh

#!/bin/bash                                                                                                                                                                                 

plugin=$1
gitRepoPlugins=$2
gitBranch=$3
gitFolder=$4
buildFolder=$5
pluginFolder=$6
publishArchitecture=$7

### CREATE FOLDERS ###
if [ -d "$gitFolder" ]
then
    echo "Directory $gitFolder exists. Removing."
    rm -r $gitFolder
fi

echo "Creating $gitFolder directory."
mkdir -p -- "$gitFolder"


if [ -d "$gitFolder/application" ]
then
    echo "Application GIT folder exists. Removing"
    rm -r "$gitFolder/application"
fi

echo "Creating GIT folder."
mkdir -p -- "$gitFolder/application"

if [ -d "$gitFolder/plugins" ]
then
    echo "Plugin GIT folder exists."
    rm -r "$gitFolder/plugins"
fi

echo "Creating plugin GIT folder."
mkdir -p -- "$gitFolder/plugins"

echo "Cloning plugins GIT repo"
git clone "$gitRepoPlugins" "$gitFolder/plugins"

### CLEAR BUILD FOLDER ###
echo "Clearing build folder"
rm -rf $buildFolder && mkdir $buildFolder

### PUBLISH PLUGINS ###
echo "Publishing plugins"
echo "Reading existing plugins in $pluginFolder"

projectFilePath=$(find "$gitFolder/plugins/$plugin" -type f -name "*.csproj" -print)

if [ ${#projectFilePath} -ge 1 ]; then
    echo "Publishing $projectFilePath"    
    dotnet publish $projectFilePath -c Release -r $publishArchitecture --output $buildFolder
fi 

### MOVE PLUGINS ###
echo "Moving plugins to plugin folder"
if [ -d "$pluginFolder" ]
then
    echo "Directory $pluginFolder exists."
else
    echo "Directory $pluginFolder does not exists. Creating."
    mkdir -p -- "$pluginFolder"
fi

rsync -av -I --include "*.dll" --exclude '*' "$buildFolder/." "$pluginFolder/."

publish-helper.sh

#!/bin/bash
sh ./publish.sh GIT_REPO_URL GIT_BRANCH PATH_FOR_LOCAL_GIT_REPO PATH_TO_BUILD_FOLDER PATH_TO_APP_FOLDER PUBLISH_ARCHITECTURE

publish.sh

#!/bin/bash                                                                                                                                                                                 

gitRepoApplication=$1
gitRepoPlugins=$2
gitBranch=$3
gitFolder=$4
buildFolder=$5
appFolder=$6
pluginFolder=$7
publishArchitecture=$8

### CREATE FOLDERS ###
if [ -d "$gitFolder" ]
then
    echo "Directory $gitFolder exists. Removing."
    rm -r $gitFolder
fi

echo "Creating $gitFolder directory."
mkdir -p -- "$gitFolder"


if [ -d "$gitFolder/application" ]
then
    echo "Application GIT folder exists. Removing"
    rm -r "$gitFolder/application"
fi

echo "Creating GIT folder."
mkdir -p -- "$gitFolder/application"

echo "Cloning appliction GIT repo"
git clone "$gitRepoApplication" "$gitFolder/application"


if [ -d "$gitFolder/plugins" ]
then
    echo "Plugin GIT folder exists."
    rm -r "$gitFolder/plugins"
fi

echo "Creating plugin GIT folder."
mkdir -p -- "$gitFolder/plugins"

echo "Cloning plugins GIT repo"
git clone "$gitRepoPlugins" "$gitFolder/plugins"

### UPDATE APPLICATION GIT ###
echo "Updating application GIT repository"
cd "$gitFolder/application"
git checkout $gitBranch
git pull

if [ -d "$buildFolder" ]
then
    echo "Directory $buildFolder exists."
else
    echo "Directory $buildFolder does not exists. Creating."
    mkdir -p -- "$buildFolder"
fi

echo "Clearing build folder"
rm -rf $buildFolder && mkdir $buildFolder

### PUBLISH APPLICATION ###
echo "Publishing application"
dotnet publish "$gitFolder/application/couchpotato.csproj" -c Release -r $publishArchitecture --output "$buildFolder"

### MOVE APPLICATION ###
echo "Moving published files to application folder"
if [ -d "$appFolder" ]
then
    echo "Directory $appFolder exists."
else
    echo "Directory $appFolder does not exists. Creating."
    mkdir -p -- "$appFolder"
fi
rsync -av -I --exclude='appsettings.json' "$buildFolder/." "$appFolder/."

### SET APPLICATION PERMISSIONS ###
echo "Setting application permissions"
chmod +x "$appFolder/couchpotato"

### UPDATE PLUGIN GIT ###
echo "Updating plugins GIT repository"
cd "$gitFolder/plugins"
git checkout $gitBranch
git pull

### CLEAR BUILD FOLDER ###
echo "Clearing build folder"
rm -rf $buildFolder && mkdir $buildFolder

### PUBLISH PLUGINS ###
echo "Publishing plugins"
echo "Reading existing plugins in $pluginFolder"

for pluginFile in $(find $pluginFolder -name "*.dll" -type f)
do
    filename="${pluginFile##*/}"
    filenameNoExt="${filename%.*}"
    projectFilePath=$(find "$gitFolder/plugins" -type f -name "*.csproj" -print | grep -i $filenameNoExt.csproj | head -1)

    if [ ${#projectFilePath} -ge 1 ]; then
        echo "Publishing $projectFilePath"
        dotnet publish $projectFilePath -c Release -r $publishArchitecture --output $buildFolder
    fi
    
    
done

### MOVE PLUGINS ###
echo "Moving plugins to plugin folder"
if [ -d "$pluginFolder" ]
then
    echo "Directory $pluginFolder exists."
else
    echo "Directory $pluginFolder does not exists. Creating."
    mkdir -p -- "$pluginFolder"
fi

rsync -av -I --include "*.dll" --exclude '*' "$buildFolder/." "$pluginFolder/."