Measurement.java

/*
 *  Licensed to GraphHopper and Peter Karich under one or more contributor
 *  license agreements. See the NOTICE file distributed with this work for 
 *  additional information regarding copyright ownership.
 *
 *  GraphHopper licenses this file to you under the Apache License, 
 *  Version 2.0 (the "License"); you may not use this file except in 
 *  compliance with the License. You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package com.graphhopper.matrix.util;

import com.graphhopper.GHRequest;
import com.graphhopper.GHResponse;
import com.graphhopper.GraphHopper;
import com.graphhopper.coll.GHBitSet;
import com.graphhopper.reader.DataReader;
import com.graphhopper.reader.osm.GraphHopperOSM;
import com.graphhopper.routing.*;
import com.graphhopper.routing.util.AllEdgesIterator;
import com.graphhopper.routing.weighting.Weighting;
import com.graphhopper.storage.CHGraph;
import com.graphhopper.storage.Graph;
import com.graphhopper.storage.GraphHopperStorage;
import com.graphhopper.storage.NodeAccess;
import com.graphhopper.util.*;
import com.graphhopper.util.shapes.GHPlace;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileWriter;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * @author Peter Karich
 */
public class Measurement {

    public static void main(String[] strs) {
        new Measurement().start(CmdArgs.read(strs));
    }

    private static final Logger logger = LoggerFactory.getLogger(Measurement.class);
    private final Map<String, String> properties = new TreeMap<String, String>();
    private long seed;
    private int maxNode;

    // creates properties file in the format key=value
    // Every value is one y-value in a separate diagram with an identical x-value for every Measurement.start call
    void start(CmdArgs args) {
        String graphLocation = args.get("graph.location", "");
        String propLocation = args.get("measurement.location", "");
        if (Helper.isEmpty(propLocation)) {
            propLocation = "measurement" + new SimpleDateFormat("yyyy-MM-dd_HH_mm_ss").format(new Date()) + ".properties";
        }

        seed = args.getLong("measurement.seed", 123);
        String gitCommit = args.get("measurement.gitinfo", "");
        int count = args.getInt("measurement.count", 5000);
        int countPerQuery = args.getInt("measurement.points_per_query", 10);

        GraphHopper hopper = new GraphHopperOSM() {
            @Override
            protected void prepareCH() {
                StopWatch sw = new StopWatch().start();
                super.prepareCH();
                put("prepare.time", sw.stop().getTime());
                int edges = getGraphHopperStorage().getAllEdges().getMaxId();
                Weighting weighting = getCHFactoryDecorator().getWeightings().get(0);
                int edgesAndShortcuts = getGraphHopperStorage().getGraph(CHGraph.class, weighting).getAllEdges().getMaxId();
                put("prepare.shortcuts", edgesAndShortcuts - edges);
            }

            @Override
            protected DataReader importData() throws IOException {
                StopWatch sw = new StopWatch().start();
                DataReader dr = super.importData();
                put("graph.import_time", sw.stop().getSeconds());
                return dr;
            }
        };

        hopper.init(args).
                forDesktop();
        hopper.getCHFactoryDecorator().setDisablingAllowed(true);
        hopper.getLMFactoryDecorator().setDisablingAllowed(true);
        hopper.importOrLoad();

        GraphHopperStorage g = hopper.getGraphHopperStorage();
        String vehicleStr = args.get("graph.flag_encoders", "car");

        StopWatch sw = new StopWatch().start();
        try {
            maxNode = g.getNodes();
            boolean isCH = false, isLM = false;

            if (hopper.getCHFactoryDecorator().isEnabled()) {
                isLM = false;
                isCH = true;
                printTimeOfMatrixQuery(hopper, isCH, isLM, false, count, countPerQuery, "routingCH", vehicleStr, -1);
//                printTimeOfMatrixQuery(hopper, isCH, isLM, true, count, countPerQuery, "routingCH_wo", vehicleStr, -1);
            }

            isCH = false;
            isLM = false;
//            printTimeOfMatrixQuery(hopper, isCH, isLM, false, Math.max(2, count / 20), countPerQuery, "routing", vehicleStr, -1);
//            printTimeOfSingleQueries(hopper, isCH, isLM, false, Math.max(2, count / 20), countPerQuery, "single_routing", vehicleStr, -1);
//            printTimeOfMatrixQuery(hopper, isCH, isLM, true, Math.max(2, count / 20), countPerQuery, "routing_wo", vehicleStr, -1);
            if (hopper.getLMFactoryDecorator().isEnabled()) {
                isLM = true;
                isCH = false;
                int activeLandmarks = 8;
//                printTimeOfMatrixQuery(hopper, isCH, isLM, false, count, countPerQuery, "routingLM" + activeLandmarks, vehicleStr, activeLandmarks);
//                printTimeOfMatrixQuery(hopper, isCH, isLM, true, count, countPerQuery, "routingLM" + activeLandmarks + "_wo", vehicleStr, activeLandmarks);
//                printTimeOfSingleQueries(hopper, isCH, isLM, false, count, countPerQuery, "single_routingLM" + activeLandmarks, vehicleStr, activeLandmarks);
            }

            System.gc();

            logger.info("store into " + propLocation);

        } catch (Exception ex) {
            logger.error("Problem while measurement of " + graphLocation, ex);
            put("error", ex.toString());
        } finally {
            put("measurement.gitinfo", gitCommit);
            put("measurement.count", count);
            put("measurement.seed", seed);
            put("measurement.time", sw.stop().getTime());
            System.gc();
            put("measurement.totalMB", Helper.getTotalMB());
            put("measurement.usedMB", Helper.getUsedMB());
            try {
                store(new FileWriter(propLocation), "measurement finish, "
                        + new Date().toString() + ", " + Constants.BUILD_DATE);
            } catch (IOException ex) {
                logger.error("Problem while storing properties " + graphLocation + ", " + propLocation, ex);
            }
        }
    }

    void fillAllowedEdges(AllEdgesIterator iter, GHBitSet bs) {
        bs.clear();
        while (iter.next()) {
            bs.add(iter.getEdge());
        }
    }

    void printTimeOfSingleQueries(final GraphHopper hopper,
                                  final boolean isCH, final boolean isLM,
                                  final boolean onlyWeights, int count,
                                  final int pointsPerQuery, String prefix, final String vehicle, final int almCount) {

        final Random rand = new Random(seed);
        final AtomicInteger failedCount = new AtomicInteger(0);
        final NodeAccess na = hopper.getGraphHopperStorage().getNodeAccess();

        MiniPerfTest miniPerf = new MiniPerfTest() {
            @Override
            public int doCalc(boolean warmup, int run) {
                GHRequest req = new GHRequest();
                req.setWeighting("fastest").setVehicle(vehicle);
                req.getHints().put(Parameters.Routing.CALC_POINTS, false)
                        .put(Parameters.Routing.INSTRUCTIONS, false);
                // this makes LM speed worse! why?
                // .put(Parameters.Algorithms.ASTAR_BI + ".epsilon", 1.3);

                List<GHPlace> list = new ArrayList<>();
                for (int i = 0; i < pointsPerQuery; i++) {
                    int nodeIndex = rand.nextInt(maxNode);
                    double lat = na.getLatitude(nodeIndex);
                    double lon = na.getLongitude(nodeIndex);
                    list.add(new GHPlace(lat, lon));
                }
                if (!isCH)
                    req.getHints().put(Parameters.CH.DISABLE, true);

                if (!isLM)
                    req.getHints().put(Parameters.Landmark.DISABLE, true);
                else
                    req.getHints().put(Parameters.Landmark.ACTIVE_COUNT, almCount);

                int count = 0;

                try {
                    for (int i = 0; i < pointsPerQuery; i++) {
                        for (int j = 0; j < pointsPerQuery; j++) {
                            if (i == j)
                                continue;

                            req.getPoints().clear();
                            req.addPoint(list.get(i));
                            req.addPoint(list.get(j));
                            GHResponse res = hopper.route(req);
                            count += res.getAll().size();
                            if (res.hasErrors()) {
                                throw new IllegalStateException("errors should NOT happen in Measurement! " + res.getErrors());
                            }
                        }
                    }

                } catch (Exception ex) {
                    throw new RuntimeException("Error while calculating route! request:" + req, ex);
                }

                if (!warmup) {
                    if (count < 1) {
                        failedCount.incrementAndGet();
                        return 0;
                    }
                }

                return count;
            }
        }.setIterations(count).start();

        put(prefix + ".failed_count", failedCount.get());
        print(prefix, miniPerf);
    }

    private void printTimeOfMatrixQuery(final GraphHopper hopper,
                                        final boolean isCH, final boolean isLM,
                                        final boolean onlyWeights, int count,
                                        final int pointsPerQuery, String prefix, final String vehicle, final int almCount) {

        logger.info("ch=" + isCH + ", lm=" + isLM + ", alm=" + almCount + ", counts=" + count + ", points/query=" + pointsPerQuery);

        final Graph g = hopper.getGraphHopperStorage();
        final MatrixAPI mp = new MatrixSimpleAPI(hopper);
        final AtomicInteger failedCount = new AtomicInteger(0);

        final Random rand = new Random(seed);
        final NodeAccess na = g.getNodeAccess();
        MiniPerfTest miniPerf = new MiniPerfTest() {
            @Override
            public int doCalc(boolean warmup, int run) {
                GHMRequest req = new GHMRequest();
                req.setWeighting("fastest").setVehicle(vehicle);
                if (onlyWeights) {
                    req.getHints().put("only_weights", true);
                } else {
                    req.addOutArray("times");
                }

                for (int i = 0; i < pointsPerQuery; i++) {
                    int nodeIndex = rand.nextInt(maxNode);
                    double lat = na.getLatitude(nodeIndex);
                    double lon = na.getLongitude(nodeIndex);
                    req.addPoint(new GHPlace(lat, lon));
                }

                if (!isCH)
                    req.getHints().put(Parameters.CH.DISABLE, true);

                if (!isLM)
                    req.getHints().put(Parameters.Landmark.DISABLE, true);
                else
                    req.getHints().put(Parameters.Landmark.ACTIVE_COUNT, almCount);

                int count = 0;
                MatrixResponse res;
                try {
                    res = mp.calc(req);

                    // check correctness via single routing requests
                    GHRequest singleReq = new GHRequest();
                    singleReq.setWeighting("fastest").setVehicle(vehicle);
                    singleReq.getHints().put(Parameters.Routing.CALC_POINTS, false)
                            .put(Parameters.Routing.INSTRUCTIONS, false);
                    for (int fromIdx = 0; fromIdx < pointsPerQuery; fromIdx++) {
                        for (int toIdx = 0; toIdx < pointsPerQuery; toIdx++) {
                            if (!res.hasNext())
                                throw new IllegalStateException("not available for [" + fromIdx + "," + toIdx + "]");

                            GHMResponse matrixRes = res.next();
                            if (fromIdx == toIdx)
                                continue;
                            if (matrixRes.hasErrors())
                                throw new IllegalStateException("errors should NOT happen in Measurement! " + matrixRes.getErrors());

                            singleReq.getPoints().clear();
                            singleReq.addPoint(req.getFromPoints().get(fromIdx));
                            singleReq.addPoint(req.getToPoints().get(toIdx));
                            GHResponse singleRes = hopper.route(singleReq);
                            if (singleRes.hasErrors())
                                throw new IllegalStateException("errors should NOT happen in Measurement! " + singleRes.getErrors());
                            // System.out.println(isCH + " [" + fromIdx + "," + toIdx + "]");
                            if (Math.abs(singleRes.getBest().getTime() - matrixRes.getBest().getTime()) > 1000) {
                                // TODO compare nodes of single request with matrix request
                                List<Path> paths = hopper.calcPaths(singleReq, new GHResponse());
                                throw new IllegalStateException("Too large difference for [" + fromIdx + "," + toIdx + "], expected:"
                                        + singleRes.getBest().getTime() + " vs. " + matrixRes.getBest().getTime()
                                        + ", request: " + singleReq + ", nodes:"+paths.get(0).calcNodes());
                            }
                            if (res.hasErrors()) {
                                throw new IllegalStateException("errors should NOT happen in Measurement! " + res.getErrors());
                            }
                        }
                    }
                } catch (Exception ex) {
                    throw new RuntimeException("Error while calculating route! request:" + req, ex);
                }
//                if (res.getCalculatedResults() != pointsPerQuery * pointsPerQuery)
//                    throw new IllegalStateException("Result count does not match. Expected " + pointsPerQuery + " vs " + res.getCalculatedResults());

                if (res.hasErrors())
                    throw new IllegalStateException("errors should NOT happen in Measurement! " + res.getErrors());

                if (!warmup) {
                    long resCount = res.getCalculatedResults();
                    if (resCount < 1) {
                        failedCount.incrementAndGet();
                        return 0;
                    }
                }

                // logger.info(pointsPerQuery + " vs. " + res.getNumSourceTrees());
                return res.getCalculatedResults() + count;
            }
        }.setIterations(count).start();

        count -= failedCount.get();
        put(prefix + ".failed_count", failedCount.get());
        print(prefix, miniPerf);
    }

    void print(String prefix, MiniPerfTest perf) {
        logger.info(perf.getReport());
        put(prefix + ".sum", perf.getSum());
//        put(prefix+".rms", perf.getRMS());
        put(prefix + ".min", perf.getMin());
        put(prefix + ".mean", perf.getMean());
        put(prefix + ".max", perf.getMax());
    }

    void put(String key, Object val) {
        // convert object to string to make serialization possible
        properties.put(key, "" + val);
    }

    private void store(FileWriter fileWriter, String comment) throws IOException {
        fileWriter.append("#" + comment + "\n");
        for (Entry<String, String> e : properties.entrySet()) {
            fileWriter.append(e.getKey());
            fileWriter.append("=");
            fileWriter.append(e.getValue());
            fileWriter.append("\n");
        }
        fileWriter.flush();
    }
}