Quickly Hacked version of local app deployer supporting restart of skipper to allow continuing control of processes

AppInstanceCollection.java

package org.springframework.cloud.skipper.server.domain;
import java.util.ArrayList;
import java.util.List;

/**
 * Created by bakopme on 11/04/2018.
 */
public class AppInstanceCollection {
    private List<AppInstanceDB> appInstances = new ArrayList<>();

    public List<AppInstanceDB> getAppInstances() {
        return appInstances;
    }

    public void setAppInstances(List<AppInstanceDB> appInstances) {
        this.appInstances = appInstances;
    }
}

AppInstanceDB.java

public class AppInstanceDB {
    private String deploymentId;
    private int instanceNumber;
    private String baseUrl;
    private int pid;
    private String workFile;
    private String stdout;
    private String stderr;
    private Map<String, String> attributes = new HashMap<String,String>();

    public String getDeploymentId() {
        return deploymentId;
    }

    public void setDeploymentId(String deploymentId) {
        this.deploymentId = deploymentId;
    }

    public int getInstanceNumber() {
        return instanceNumber;
    }

    public void setInstanceNumber(int instanceNumber) {
        this.instanceNumber = instanceNumber;
    }

    public String getBaseUrl() {
        return baseUrl;
    }

    public void setBaseUrl(String baseUrl) {
        this.baseUrl = baseUrl;
    }

    public int getPid() {
        return pid;
    }

    public void setPid(int pid) {
        this.pid = pid;
    }

    public String getWorkFile() {
        return workFile;
    }

    public void setWorkFile(String workFile) {
        this.workFile = workFile;
    }

    public String getStdout() {
        return stdout;
    }

    public void setStdout(String stdout) {
        this.stdout = stdout;
    }

    public String getStderr() {
        return stderr;
    }

    public void setStderr(String stderr) {
        this.stderr = stderr;
    }

    public Map<String, String> getAttributes() {
        return attributes;
    }

    public void setAttributes(Map<String, String> attributes) {
        this.attributes = attributes;
    }
}

AppInstanceDeploymentData.java

/*
 * Copyright 2017-2018 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.springframework.cloud.skipper.server.domain;

import org.springframework.cloud.skipper.domain.AbstractEntity;

import javax.persistence.*;
import javax.validation.constraints.NotNull;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * The entity corresponds to an app instance that has been deployed.
 *
 * @author Menelaos Bakopoulos
 */
@Entity
@Table(name = "AppInstanceDeploymentData", indexes = @Index(name="deploymentId", columnList = "deploymentId"))
public class AppInstanceDeploymentData extends AbstractEntity {

	@NotNull private String deploymentId;
	@NotNull private String AppInstanceCollectionJson;

	public String getDeploymentId() {
		return deploymentId;
	}

	public void setDeploymentId(String deploymentId) {
		this.deploymentId = deploymentId;
	}

	public String getAppInstanceCollectionJson() {
		return AppInstanceCollectionJson;
	}

	public void setAppInstanceCollectionJson(String appInstanceCollectionJson) {
		AppInstanceCollectionJson = appInstanceCollectionJson;
	}
}

SGLocalAppDeployer.java

//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by Fernflower decompiler)
//

package org.springframework.cloud.skipper.server.deployer;

import java.io.*;
import java.lang.ProcessBuilder.Redirect;
import java.lang.reflect.Field;
import java.net.HttpURLConnection;
import java.net.Inet4Address;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileAttribute;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import javax.annotation.PreDestroy;
import javax.validation.constraints.Null;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cloud.deployer.spi.app.AppDeployer;
import org.springframework.cloud.deployer.spi.app.AppInstanceStatus;
import org.springframework.cloud.deployer.spi.app.AppStatus;
import org.springframework.cloud.deployer.spi.app.DeploymentState;
import org.springframework.cloud.deployer.spi.app.AppStatus.Builder;
import org.springframework.cloud.deployer.spi.core.AppDeploymentRequest;
import org.springframework.cloud.deployer.spi.core.RuntimeEnvironmentInfo;
import org.springframework.cloud.deployer.spi.local.AbstractLocalDeployerSupport;
import org.springframework.cloud.deployer.spi.local.LocalDeployerProperties;
import org.springframework.cloud.skipper.server.domain.AppInstanceCollection;
import org.springframework.cloud.skipper.server.domain.AppInstanceDB;
import org.springframework.cloud.skipper.server.domain.AppInstanceDeploymentData;
import org.springframework.cloud.skipper.server.repository.AppInstanceRepository;
import org.springframework.http.ResponseEntity;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;

public class SGLocalAppDeployer extends AbstractLocalDeployerSupport implements AppDeployer{
    private Path logPathRoot;
    private static final Logger logger = LoggerFactory.getLogger(SGLocalAppDeployer.class);
    private static final String JMX_DEFAULT_DOMAIN_KEY = "spring.jmx.default-domain";
    private static final String ENDPOINTS_SHUTDOWN_ENABLED_KEY = "endpoints.shutdown.enabled";

    private final Map<String, List<SGLocalAppDeployer.AppInstance>> running = new ConcurrentHashMap();
    private static AppInstanceRepository appInstanceRepository;
    private ObjectMapper objectMapper = new ObjectMapper();


    public SGLocalAppDeployer(LocalDeployerProperties properties, AppInstanceRepository appInstanceRepository) {
        super(properties);

        this.appInstanceRepository = appInstanceRepository;
        try {
            this.logPathRoot = Files.createTempDirectory("skipperRunningMetadata");

        } catch (IOException var3) {
            throw new RuntimeException("Could not create workdir root: " + properties.getWorkingDirectoriesRoot(), var3);
        }

        loadPersistedAppInstances(appInstanceRepository);
    }

    public String deploy(AppDeploymentRequest request) {
        String group = (String)request.getDeploymentProperties().get("spring.cloud.deployer.group");
        String deploymentId = String.format("%s.%s", new Object[]{group, request.getDefinition().getName()});
        this.validateStatus(deploymentId);
        List<SGLocalAppDeployer.AppInstance> processes = new ArrayList();
        this.running.put(deploymentId, processes);
        boolean useDynamicPort = !request.getDefinition().getProperties().containsKey("server.port");
        HashMap<String, String> consolidatedAppProperties = new HashMap(request.getDefinition().getProperties());
        consolidatedAppProperties.put("spring.jmx.default-domain", deploymentId);
        if(!request.getDefinition().getProperties().containsKey("endpoints.shutdown.enabled")) {
            consolidatedAppProperties.put("endpoints.shutdown.enabled", "true");
        }

        consolidatedAppProperties.put("endpoints.jmx.unique-names", "true");
        if(group != null) {
            consolidatedAppProperties.put("spring.cloud.application.group", group);
        }

        try {
            Path deploymentGroupDir = this.createLogDir(group);
            Path workDir = this.createWorkingDir(deploymentId, deploymentGroupDir);
            String countProperty = (String)request.getDeploymentProperties().get("spring.cloud.deployer.count");
            int count = StringUtils.hasText(countProperty)?Integer.parseInt(countProperty):1;

            for(int i = 0; i < count; ++i) {
                Map<String, String> appInstanceEnv = new HashMap(consolidatedAppProperties);
                int port = this.calcServerPort(request, useDynamicPort, appInstanceEnv);
                appInstanceEnv.put("INSTANCE_INDEX", Integer.toString(i));
                appInstanceEnv.put("SPRING_APPLICATION_INDEX", Integer.toString(i));
                appInstanceEnv.put("SPRING_CLOUD_APPLICATION_GUID", Integer.toString(port));
                SGLocalAppDeployer.AppInstance instance = new SGLocalAppDeployer.AppInstance(deploymentId, i, port);
                ProcessBuilder builder = this.buildProcessBuilder(request, appInstanceEnv, Optional.of(Integer.valueOf(i)), deploymentId).inheritIO();
                builder.directory(workDir.toFile());
                if(this.shouldInheritLogging(request)) {
                    instance.start(builder, workDir);
                    logger.info("Deploying app with deploymentId {} instance {}.\n   Logs will be inherited.", deploymentId, Integer.valueOf(i));
                } else {
                    instance.start(builder, workDir, this.getLocalDeployerProperties().isDeleteFilesOnExit());
                    logger.info("Deploying app with deploymentId {} instance {}.\n   Logs will be in {}", new Object[]{deploymentId, Integer.valueOf(i), workDir});
                }

                processes.add(instance);
            }

            save(deploymentId, processes);

            return deploymentId;
        } catch (IOException var16) {
            throw new RuntimeException("Exception trying to deploy " + request, var16);
        }
    }

    private final RestTemplate restTemplate = new RestTemplate();

    protected void shutdownAndWait(AbstractLocalDeployerSupport.Instance instance) {
        try {
            int timeout = this.getLocalDeployerProperties().getShutdownTimeout();
            if(timeout > 0) {
                ResponseEntity<String> response = this.restTemplate.postForEntity(instance.getBaseUrl() + "/shutdown", (Object)null, String.class, new Object[0]);
                if(response.getStatusCode().is2xxSuccessful()) {
                    long timeoutTimestamp = System.currentTimeMillis() + (long)(timeout * 1000);

                    while(isAlive(instance.getProcess()) && System.currentTimeMillis() < timeoutTimestamp) {
                        Thread.sleep(1000L);
                    }
                }
            }
        } catch (InterruptedException var10) {
            Thread.currentThread().interrupt();
        } catch (Exception var11) {
            ;
        } finally {
            if(isAlive(instance.getProcess())) {
                if(instance.getProcess() != null){
                    instance.getProcess().destroy();
                }
            }

        }

    }


    public void undeploy(String id) {
        List<SGLocalAppDeployer.AppInstance> processes = (List)this.running.get(id);

        if(processes != null) {
            Iterator var3 = processes.iterator();

            while(var3.hasNext()) {
                SGLocalAppDeployer.AppInstance instance = (SGLocalAppDeployer.AppInstance)var3.next();
                if(isAlive(instance.getProcess())) {
                    logger.info("Un-deploying app with deploymentId {} instance {}.", id, Integer.valueOf(instance.getInstanceNumber()));
                    this.shutdownAndWait(instance);
                }
            }

            this.running.remove(id);
            AppInstanceDeploymentData appInstanceDB = new AppInstanceDeploymentData();
            appInstanceDB.setDeploymentId(id);
            appInstanceDB.setAppInstanceCollectionJson("");
            appInstanceRepository.delete(appInstanceDB);

        } else {
            logger.info(String.format("App with deploymentId %s is not in a deployed state.", new Object[]{id}));
            AppInstanceDeploymentData appInstanceDB = new AppInstanceDeploymentData();
            appInstanceDB.setDeploymentId(id);
            appInstanceDB.setAppInstanceCollectionJson("");
            appInstanceRepository.delete(appInstanceDB);
        }
    }


    protected static boolean isAlive(Process process) {
        try {

            if(process == null){
                //TODO: Create a special endpoint in all our deployed instances to have a special random deployment ID
                // or use eureka to check if they are alive, since we don't have access to process object.
                return true;
            }

            process.exitValue();
            return false;
        } catch (IllegalThreadStateException var2) {
            return true;
        }
    }

    public AppStatus status(String id) {
        List<SGLocalAppDeployer.AppInstance> instances = (List)this.running.get(id);
        Builder builder = AppStatus.of(id);
        if(instances != null) {
            Iterator var4 = instances.iterator();

            while(var4.hasNext()) {
                SGLocalAppDeployer.AppInstance instance = (SGLocalAppDeployer.AppInstance)var4.next();
                builder.with(instance);
            }
        }

        return builder.build();
    }

    public RuntimeEnvironmentInfo environmentInfo() {
        return super.createRuntimeEnvironmentInfo(AppDeployer.class, this.getClass());
    }

    @PreDestroy
    public void shutdown() throws Exception {
        Iterator var1 = this.running.keySet().iterator();

        while(var1.hasNext()) {
            String deploymentId = (String)var1.next();
            this.undeploy(deploymentId);
        }

    }

    private Path createWorkingDir(String deploymentId, Path deploymentGroupDir) throws IOException {
        Path workDir = Files.createDirectory(Paths.get(deploymentGroupDir.toFile().getAbsolutePath(), new String[]{deploymentId}), new FileAttribute[0]);
        if(this.getLocalDeployerProperties().isDeleteFilesOnExit()) {
            workDir.toFile().deleteOnExit();
        }

        return workDir;
    }

    private Path createLogDir(String group) throws IOException {
        Path deploymentGroupDir = Paths.get(this.logPathRoot.toFile().getAbsolutePath(), new String[]{group + "-" + System.currentTimeMillis()});
        if(!Files.exists(deploymentGroupDir, new LinkOption[0])) {
            Files.createDirectory(deploymentGroupDir, new FileAttribute[0]);
            deploymentGroupDir.toFile().deleteOnExit();
        }

        return deploymentGroupDir;
    }

    private void validateStatus(String deploymentId) {
        DeploymentState state = this.status(deploymentId).getState();
        Assert.state(state == DeploymentState.unknown, String.format("App with deploymentId [%s] is already deployed with state [%s]", new Object[]{deploymentId, state}));
    }

    private boolean shouldInheritLogging(AppDeploymentRequest request) {
        boolean inheritLogging = false;
        if(request.getDeploymentProperties().containsKey("spring.cloud.deployer.local.inheritLogging")) {
            inheritLogging = Boolean.parseBoolean((String)request.getDeploymentProperties().get("spring.cloud.deployer.local.inheritLogging"));
        }

        return inheritLogging;
    }

    private static Integer getProcessExitValue(Process process) {
        try {
            return Integer.valueOf(process.exitValue());
        } catch (IllegalThreadStateException var2) {
            return null;
        }
    }

    private static synchronized int getLocalProcessPid(Process p) {
        int pid = 0;

        try {
            if(p.getClass().getName().equals("java.lang.UNIXProcess")) {
                Field f = p.getClass().getDeclaredField("pid");
                f.setAccessible(true);
                pid = f.getInt(p);
                f.setAccessible(false);
            }
        } catch (Exception var3) {
            pid = 0;
        }

        return pid;
    }



    private static class AppInstance implements Instance, AppInstanceStatus {
        private final String deploymentId;
        private final int instanceNumber;
        private final URL baseUrl;
        private int pid;
        private Process process;
        private File workFile;
        private File stdout;
        private File stderr;
        private final Map<String, String> attributes;

        private AppInstance(String deploymentId, int instanceNumber, int port) throws IOException {
            this.attributes = new TreeMap();
            this.deploymentId = deploymentId;
            this.instanceNumber = instanceNumber;
            this.attributes.put("port", Integer.toString(port));
            this.attributes.put("guid", Integer.toString(port));
            this.baseUrl = new URL("http", Inet4Address.getLocalHost().getHostAddress(), port, "");
            this.attributes.put("url", this.baseUrl.toString());
        }

        public String getId() {
            return this.deploymentId + "-" + this.instanceNumber;
        }

        public URL getBaseUrl() {
            return this.baseUrl;
        }

        public Process getProcess() {
            return this.process;
        }

        public String toString() {
            return String.format("%s [%s]", new Object[]{this.getId(), this.getState()});
        }

        public DeploymentState getState() {

            if(this.process == null){   //We have lost the process object but this has clearly already been deployed

                try {
                    HttpURLConnection urlConnection = (HttpURLConnection)this.baseUrl.openConnection();
                    urlConnection.setConnectTimeout(100);
                    urlConnection.connect();
                    urlConnection.disconnect();
                    return DeploymentState.deployed;
                } catch (IOException var3) {
                    return DeploymentState.failed;
                }

            }else{
                Integer exit = SGLocalAppDeployer.getProcessExitValue(this.process);
                if(exit != null) {
                    return DeploymentState.failed;
                } else {
                    try {
                        HttpURLConnection urlConnection = (HttpURLConnection)this.baseUrl.openConnection();
                        urlConnection.setConnectTimeout(100);
                        urlConnection.connect();
                        urlConnection.disconnect();
                        return DeploymentState.deployed;
                    } catch (IOException var3) {
                        return DeploymentState.deploying;
                    }
                }
            }

        }

        public int getInstanceNumber() {
            return this.instanceNumber;
        }

        public Map<String, String> getAttributes() {
            return this.attributes;
        }

        private void start(ProcessBuilder builder, Path workDir) throws IOException {
            this.workFile = workDir.toFile();
            this.attributes.put("working.dir", this.workFile.getAbsolutePath());
            this.process = builder.start();
            this.pid = SGLocalAppDeployer.getLocalProcessPid(this.process);
            if(this.pid > 0) {
                this.attributes.put("pid", Integer.toString(this.pid));
            }

        }

        private void start(ProcessBuilder builder, Path workDir, boolean deleteOnExist) throws IOException {
            String workDirPath = workDir.toFile().getAbsolutePath();
            this.stdout = Files.createFile(Paths.get(workDirPath, new String[]{"stdout_" + this.instanceNumber + ".log"}), new FileAttribute[0]).toFile();
            this.attributes.put("stdout", this.stdout.getAbsolutePath());
            this.stderr = Files.createFile(Paths.get(workDirPath, new String[]{"stderr_" + this.instanceNumber + ".log"}), new FileAttribute[0]).toFile();
            this.attributes.put("stderr", this.stderr.getAbsolutePath());
            if(deleteOnExist) {
                this.stdout.deleteOnExit();
                this.stderr.deleteOnExit();
            }

            builder.redirectOutput(Redirect.to(this.stdout));
            builder.redirectError(Redirect.to(this.stderr));
            this.start(builder, workDir);
        }
    }


    private void loadPersistedAppInstances(AppInstanceRepository appInstanceRepository) {
        Iterable<AppInstanceDeploymentData> iterable  = appInstanceRepository.findAll();
        Iterator<AppInstanceDeploymentData> iterator = iterable.iterator();

        while(iterator.hasNext()){
            AppInstanceDeploymentData deploymentData = iterator.next();
            String JSON = deploymentData.getAppInstanceCollectionJson();
            try {
                AppInstanceCollection appInstanceCollection = objectMapper.readValue(JSON, AppInstanceCollection.class);
                List<AppInstanceDB> appInstanceDBS = appInstanceCollection.getAppInstances();

                Iterator<AppInstanceDB> dbIterator = appInstanceDBS.iterator();

                List<AppInstance> processes = new ArrayList();
                while(dbIterator.hasNext()){
                    AppInstanceDB db = dbIterator.next();

                    AppInstance appInstance = new AppInstance(db.getDeploymentId(), db.getInstanceNumber(), Integer.parseInt(db.getAttributes().get("port")));
                    appInstance.pid = db.getPid();
                    try{
                        appInstance.workFile = new File(db.getWorkFile());
                        appInstance.stdout = new File(db.getStdout());
                        appInstance.stderr = new File(db.getStderr());
                    }catch (NullPointerException e){

                    }

                    Iterator<Entry<String, String>> att = db.getAttributes().entrySet().iterator();
                    Map<String, String> ramAttributes = appInstance.getAttributes();
                    while(att.hasNext()){
                        Entry<String, String> attrib = att.next();
                        ramAttributes.put(attrib.getKey(), attrib.getValue());
                    }

                    processes.add(appInstance);
                }

                //TODO: Force a check of the status here in order to know what is working and what has failed.
                this.running.put(deploymentData.getDeploymentId(), processes);

            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }


    private void save(String deploymentId, List<AppInstance> processes) throws JsonProcessingException {
        AppInstanceCollection appInstanceCollection = new AppInstanceCollection();
        Iterator<AppInstance> a = processes.iterator();
        while(a.hasNext()){
            AppInstance deployedInstance = a.next();

            AppInstanceDB appInstanceData = new AppInstanceDB();
            appInstanceData.setDeploymentId(deployedInstance.deploymentId);
            appInstanceData.setInstanceNumber(deployedInstance.instanceNumber);
            appInstanceData.setBaseUrl(deployedInstance.getBaseUrl().toExternalForm());
            appInstanceData.setPid(deployedInstance.pid);
            appInstanceData.setWorkFile(deployedInstance.workFile.getAbsolutePath());
            appInstanceData.setStderr(deployedInstance.stdout.getAbsolutePath());
            appInstanceData.setStdout(deployedInstance.stderr.getAbsolutePath());
            appInstanceData.setAttributes(deployedInstance.getAttributes());

            appInstanceCollection.getAppInstances().add(appInstanceData);
        }

        AppInstanceDeploymentData appInstanceDeploymentData = new AppInstanceDeploymentData();
        appInstanceDeploymentData.setDeploymentId(deploymentId);
        appInstanceDeploymentData.setAppInstanceCollectionJson(objectMapper.writeValueAsString(appInstanceCollection));

        appInstanceRepository.save(appInstanceDeploymentData);
    }
}