parse_es_migration.java

package org.sonar.ce.task.projectanalysis.step;

import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import javax.annotation.CheckForNull;
import javax.annotation.Nullable;
import org.junit.Test;

import static java.lang.String.format;
import static org.assertj.core.api.Assertions.assertThat;

public class ReadEsAsciiDoc {
  @Test
  public void readMigrations() throws IOException {
    Map<String, Map<String, List<MigrationInfo>>> res = new LinkedHashMap<>();
    Stream.of(
      readMigrations("=", "6.6.0", Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/migration/migrate_6_6.asciidoc")),
      readMigrations("=", "6.5.0", Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/migration/migrate_6_5.asciidoc")),
      readMigrations("=", "6.4.0", Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/migration/migrate_6_4.asciidoc")),
      readMigrations("=", "6.3.0", Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/migration/migrate_6_3.asciidoc")),
      readMigrations("", "6.2.0", Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/migration/migrate_6_2.asciidoc")),
      readMigrations("", "6.1.0", Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/migration/migrate_6_1.asciidoc")),
      read60Migrations()
      )
      .map(t -> t.entrySet().stream())
      .flatMap(t -> t)
      .forEach(t -> res.put(t.getKey(), t.getValue()));

    assertThat(res).isNotEmpty();

    String pattern = "\"%s\",\"%s\",\"%s\",\"%s\"";
    List<String> csvLines = new ArrayList<>();
    csvLines.add(format(pattern, "version", "category", "title", "description"));
    res.forEach((version, map) -> {
      csvLines.add(format(pattern, version, "", "", ""));
      map.forEach(((category, dataInfos) -> {
        dataInfos.forEach(dataInfo -> {
          String data = dataInfo.data == null ? "" : dataInfo.data.replaceAll("\"", "\"\"");
          csvLines.add(format(pattern, version, category, dataInfo.title, data));
        });
      }));
    });
    Files.write(Paths.get("/tmp/es_migration.csv"), csvLines);
  }

  private Map<String, Map<String, List<MigrationInfo>>> read60Migrations() throws IOException {
    Iterable<Path> files = () -> {
      try {
        return Files.newDirectoryStream(Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/migration/migrate_6_0")).iterator();
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    };
    return readMigrations("=", "6.0.0", StreamSupport.stream(files.spliterator(), false).toArray(Path[]::new));
  }

  private static Map<String, Map<String, List<MigrationInfo>>> readMigrations(String titlePrefix, String version, Path... paths) throws IOException {
    MigrationAsciidocReader reader = new MigrationAsciidocReader(titlePrefix, version);
    for (Path path : paths) {
      Files.lines(path, Charset.defaultCharset()).forEach(reader);
    }
    reader.flush();
    return reader.output;
  }

  private static final class MigrationInfo {
    private final String title;
    @CheckForNull
    private final String data;

    private MigrationInfo(String title, @Nullable String data) {
      this.title = title;
      this.data = data;
    }
  }

  private static class MigrationAsciidocReader implements Consumer<String> {
    private final String titlePrefix;
    private final String version;
    private final Map<String, Map<String, List<MigrationInfo>>> output = new LinkedHashMap<>();
    private int lineNumber = 0;
    private String currentCategory;
    private String currentTitle;
    private String currentData;

    private MigrationAsciidocReader(String titlePrefix, String version) {
      this.titlePrefix = titlePrefix;
      this.version = version;
    }

    @Override
    public void accept(String line) {
      lineNumber++;

      if (isAnchorOrFormatting(line) || isComment(line)) {
        return;
      }

      Optional<String> category = readCategory(line);
      if (category.isPresent()) {
        flush();
        currentCategory = category.get();
        currentTitle = null;
        currentData = null;
        return;
      }

      Optional<String> title = readTitle(line);
      if (title.isPresent()) {
        flush();
        currentTitle = title.get();
        currentData = null;
        return;
      }

      if (currentTitle != null && !line.isEmpty() && currentData == null) {
        currentData = line;
        printData();
        return;
      }

      if (currentData != null) {
        currentData += '\n' + line;
        printData();
        return;
      }

      if (!line.isEmpty()) {
        System.err.println(format("??? %s : %s : %s : %s", version, currentCategory, currentTitle, line));
      }
    }

    private void flush() {
      if (currentCategory != null && currentTitle != null) {
        output.compute(version,
          (version, existingMap) -> {
            Map<String, List<MigrationInfo>> value = existingMap == null ? new LinkedHashMap<>() : existingMap;
            value.compute(currentCategory,
              (category, existingList) -> {
                List<MigrationInfo> list = existingList == null ? new ArrayList<>() : existingList;
                list.add(new MigrationInfo(currentTitle, currentData));
                return list;
              });
            return value;
          });
        currentTitle = null;
        currentData = null;
      }
    }

    private void printData() {
//      System.out.println(format("(%s) DATA %s : %s : %s : %s", lineNumber, version, currentCategory, currentTitle, currentData));
    }

    private Optional<String> readCategory(String line) {
      String prefix = titlePrefix + "== ";
      if (line.startsWith(prefix)) {
        return Optional.of(line.substring(prefix.length()));
      }
      return Optional.empty();
    }

    private Optional<String> readTitle(String line) {
      String prefix = titlePrefix + "=== ";
      if (line.startsWith(prefix)) {
        return Optional.of(line.substring(prefix.length()));
      }
      return Optional.empty();
    }
  }

  @Test
  public void readBreakingChanges() throws IOException {
    Map<String, Map<ChangeType, List<DataInfo>>> output = readBreakingChanges(
      Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/release-notes/6.6.asciidoc"),
      Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/release-notes/6.5.asciidoc"),
      Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/release-notes/6.4.asciidoc"),
      Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/release-notes/6.3.asciidoc"),
      Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/release-notes/6.2.asciidoc"),
      Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/release-notes/6.1.asciidoc"),
      Paths.get("/home/sebastienl/DEV/elasticsearch/docs/reference/release-notes/6.0.asciidoc"));
    assertThat(output).isNotEmpty();

    String pattern = "\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"";
    List<String> csvLines = new ArrayList<>();
    csvLines.add(format(pattern, "version", "changeType", "category", "status", "description"));
    output.forEach((version, map) -> {
      csvLines.add(format(pattern, version, "", "", "", ""));
      map.forEach(((changeType, dataInfos) -> {
        dataInfos.forEach(dataInfo -> {
          String description = dataInfo.info + (dataInfo.subInfo == null ? "" : dataInfo.subInfo);
          csvLines.add(format(pattern, version, changeType, dataInfo.category, "", description));
        });
      }));
    });
    Files.write(Paths.get("/tmp/es_changes.csv"), csvLines);
  }

  private Map<String, Map<ChangeType, List<DataInfo>>> readBreakingChanges(Path... inputs) throws IOException {
    BreakingChangeAsciidocReader reader = new BreakingChangeAsciidocReader();
    for (Path input : inputs) {
      Files.lines(input, Charset.defaultCharset()).forEach(reader);
    }
    reader.flush(false);
    return reader.output;
  }

  private enum ChangeType {
    BREAKING_CHANGES("Breaking Changes"),
    JAVA_BREAKING_CHANGES("Breaking Java Changes"),
    IGNORED(null);

    @CheckForNull
    private final String text;

    ChangeType(@Nullable String text) {
      this.text = text;
    }

    public static Optional<ChangeType> fromLine(String line) {
      String prefix = "=== ";
      if (!line.startsWith(prefix)) {
        return Optional.empty();
      }
      Optional<ChangeType> supported = Arrays.stream(values())
        .filter(t -> t.text != null)
        .filter(t -> line.equalsIgnoreCase(prefix + t.text))
        .findFirst();
      if (supported.isPresent()) {
        return supported;
      }
      return Optional.of(IGNORED);
    }
  }

  private static final class DataInfo {
    private final String category;
    private final String info;
    @CheckForNull
    private final String subInfo;

    private DataInfo(String category, String info, @Nullable String subInfo) {
      this.category = category;
      this.info = info;
      this.subInfo = subInfo;
    }

    @Override
    public String toString() {
      if (subInfo == null) {
        return category + ":: " + info;
      }
      return category + " :: " + info + " : " + subInfo;
    }
  }

  private static class BreakingChangeAsciidocReader implements Consumer<String> {
    private final Map<String, Map<ChangeType, List<DataInfo>>> output = new LinkedHashMap<>();
    private int lineNumber = 0;
    private String currentVersion = null;
    private ChangeType currentChangeType = null;
    private String currentCategory = null;
    private String currentInfo = null;
    private String currentSubInfo = null;

    @Override
    public void accept(String line) {
      lineNumber++;

      if (isAnchorOrFormatting(line)) {
        flush(false);
        return;
      }

      Optional<String> version = readVersion(line);
      if (version.isPresent()) {
        flush(false);
        currentVersion = version.get();
        currentChangeType = null;
        currentCategory = null;
        currentInfo = null;
        currentSubInfo = null;
        return;
      }

      Optional<ChangeType> changeType = currentVersion == null ? Optional.empty() : ChangeType.fromLine(line);
      if (changeType.isPresent()) {
        currentChangeType = changeType.filter(t -> t != ChangeType.IGNORED).orElse(null);
        currentCategory = null;
        currentInfo = null;
        currentSubInfo = null;
        return;
      }

      Optional<String> category = currentChangeType == null ? Optional.empty() : readCategory(line);
      if (category.isPresent()) {
        currentCategory = category.get();
        currentInfo = null;
        currentSubInfo = null;
        return;
      }

      Optional<String> info = currentCategory == null ? Optional.empty() : readInfo(line);
      if (info.isPresent()) {
        flush(false);
        currentInfo = info.get();
        currentSubInfo = null;
        printData();
        return;
      }

      Optional<String> subInfo = currentInfo == null ? Optional.empty() : readSubInfo(line);
      if (subInfo.isPresent()) {
        flush(true);
        currentSubInfo = subInfo.get();
        printData();
        return;
      }

      if (!line.isEmpty()) {
        if (currentSubInfo != null) {
          currentSubInfo += " " + line;
          printData();
          return;
        }
        if (currentInfo != null) {
          currentInfo += " " + line;
          printData();
          return;
        }
      } else {
        flush(false);
      }

      System.err.println(format("??? %s : %s : %s : %s", currentVersion, currentChangeType, currentCategory, line));
    }

    private void flush(boolean onsubInfoOnly) {
      if (currentVersion != null && currentChangeType != null && currentCategory != null && currentInfo != null
        && (!onsubInfoOnly || currentSubInfo != null)) {
        output.compute(currentVersion,
          (version, existingMap) -> {
            Map<ChangeType, List<DataInfo>> value = existingMap == null ? new LinkedHashMap<>() : existingMap;
            value.compute(currentChangeType,
              ((changeType, existingList) -> {
                List<DataInfo> list = existingList == null ? new ArrayList<>() : existingList;
                list.add(new DataInfo(currentCategory, currentInfo, currentSubInfo));
                return list;
              }));
            return value;
          });
        if (!onsubInfoOnly) {
          currentInfo = null;
        }
        currentSubInfo = null;
      }
    }

    private void printData() {
      System.out.println(format("(%s) DATA %s : %s : %s : %s : %s", lineNumber, currentVersion, currentChangeType, currentCategory, currentInfo, currentSubInfo));
    }

    private Optional<String> readSubInfo(String line) {
      if (line.startsWith("** ")) {
        return Optional.of(line.substring("** ".length()));
      }
      return Optional.empty();
    }

    private Optional<String> readInfo(String line) {
      if (line.startsWith("* ")) {
        return Optional.of(line.substring(2));
      }
      return Optional.empty();
    }

    private Optional<String> readVersion(String line) {
      String prefix = "== {es} version ";
      if (line.startsWith(prefix)) {
        return Optional.of(line.substring(prefix.length()));
      }
      return Optional.empty();
    }

    private Optional<String> readCategory(String line) {
      if (line.endsWith("::")) {
        return Optional.of(line.substring(0, line.length() - "::".length()));
      }
      return Optional.empty();
    }
  }

  private static boolean isAnchorOrFormatting(String line) {
    return line.startsWith("[");
  }

  private static boolean isComment(String line) {
    return line.startsWith("// ");
  }
}