ReadEsAsciiDoc.java

import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import static java.lang.String.format;

/**
 * How to use:
 *  1. clone Elasticsearch source repository (https://github.com/elastic/elasticsearch)
 *     and checkout the branch or tag of the version you intend to upgrade to
 *  2. create java file ReadEsAsciiDoc.java with this source
 *  3. update paths below to the release notes and migration asciidoc files you want to parse
 *     (see methods readRelaseNotes and readMigrations)
 *  4. compile and execute with: javac ReadEsAsciiDoc.java && java ReadEsAsciiDoc [elasticsearch source clone location]
 *  5. import CVS into a Google sheet with the import feature (Files > Import) choosing the following options:
 *      Import location: new sheet
 *      Separator type: comma
 *      Convert text to numbers, dates and formulas: Yes
 *
 *  debug output of the parsing is logged to stdout
 *
 *  CVS files produced are:
 *    * /tmp/es_changes.csv
 *    * /tmp/es_migration.csv
 */
public class ReadEsAsciiDoc {
  private final String esSourceLocation;

  public ReadEsAsciiDoc(String esSourceLocation) {
    this.esSourceLocation = esSourceLocation;
  }

  public static void main(String[] args) throws IOException {
    if (args.length != 1) {
      throw new IllegalArgumentException("path to clone of Elasticsearch source code must be provided");
    }

    ReadEsAsciiDoc it = new ReadEsAsciiDoc(args[0]);
    it.readRelaseNotes();
    it.readMigrations();
  }

//  @VisibleForTesting
  public void readRelaseNotes() throws IOException {
    String releaseNotesPath = esSourceLocation + "/docs/reference/release-notes";
    Map<String, Map<ChangeType, List<DataInfo>>> output = readRelaseNotes(
      Paths.get(releaseNotesPath + "/7.4.asciidoc"),
      Paths.get(releaseNotesPath + "/7.3.asciidoc"),
      Paths.get(releaseNotesPath + "/7.2.asciidoc"),
      Paths.get(releaseNotesPath + "/7.1.asciidoc"),
      Paths.get(releaseNotesPath + "/7.0.asciidoc"));
    if (output.isEmpty()) {
      throw new IllegalStateException("no release note read");
    }

    String pattern = "\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"";
    List<String> csvLines = new ArrayList<>();
    csvLines.add(format(pattern, "version", "changeType", "category", "status", "description", "Pull Request", "Issues"));
    output.forEach((version, map) -> {
      map.forEach(((changeType, dataInfos) -> {
        dataInfos.forEach(dataInfo -> {
          ReleaseNoteDescription description = parseDescription(dataInfo);
          csvLines.add(
            format(pattern,
              version, changeType, dataInfo.category, "",
              googleSheetEscape(description.text), linkToPullRequest(description), linkToFirstIssue(description))
              // add extra cells (not declared in header) if there is more than one issue
              + linkToOtherIssues(description));
        });
      }));
    });
    Files.write(Paths.get("/tmp/es_changes.csv"), csvLines);
  }

  //  @VisibleForTesting
  public void readMigrations() throws IOException {
    String migrationNotesPath = esSourceLocation + "/docs/reference/migration";
    Map<String, Map<String, List<MigrationInfo>>> res = new LinkedHashMap<>();
    Stream.of(
      readMigrations("=", "7.4.0", Paths.get(migrationNotesPath + "/migrate_7_4.asciidoc")),
      readMigrations("=", "7.3.0", Paths.get(migrationNotesPath + "/migrate_7_3.asciidoc")),
      readMigrations("", "7.2.0", Paths.get(migrationNotesPath + "/migrate_7_2.asciidoc")),
      readMigrations("", "7.1.0", Paths.get(migrationNotesPath + "/migrate_7_1.asciidoc")),
      readMigrations("=", "7.0.0", Paths.get(migrationNotesPath + "/migrate_7_0")))
      .map(t -> t.entrySet().stream())
      .flatMap(t -> t)
      .forEach(t -> res.put(t.getKey(), t.getValue()));
    if (res.isEmpty()) {
      throw new IllegalStateException("no migration read");
    }

    String pattern = "\"%s\",\"%s\",\"%s\",\"%s\",\"%s\"";
    List<String> csvLines = new ArrayList<>();
    csvLines.add(format(pattern, "version", "category", "title", "status", "description"));
    res.forEach((version, map) -> {
      map.forEach(((category, dataInfos) -> {
        dataInfos.forEach(dataInfo -> {
          String data = dataInfo.data == null ? "" : dataInfo.data.replaceAll("\"", "\"\"");
          csvLines.add(format(pattern, version, category, dataInfo.title, "", googleSheetEscape(data)));
        });
      }));
    });
    Files.write(Paths.get("/tmp/es_migration.csv"), csvLines);
  }

  private static final class ReleaseNoteDescription {
    private static final int[] NO_ISSUE_NUMBER = new int[0];
    private final String text;
    private final Integer prNumber;
    private final int[] issueNumbers;

    private ReleaseNoteDescription(String text, Integer prNumber, int[] issueNumbers) {
      this.text = text;
      this.prNumber = prNumber;
      this.issueNumbers = issueNumbers;
    }
  }

  private ReleaseNoteDescription parseDescription(DataInfo dataInfo) {
    String text = dataInfo.info + (dataInfo.subInfo == null ? "" : dataInfo.subInfo);
    Integer prNumber = parsePullRequestNumber(text);
    int[] issueNumbers = parseIssueNumbers(text);
    return new ReleaseNoteDescription(text, prNumber, issueNumbers);
  }

  /**
   * Extract Pull Request number from a string which looks like
   * {@code Improve progress reporting for data frame analytics {pull}45856[#45856]}.
   */
  private static Integer parsePullRequestNumber(String description) {
    if (description.isEmpty()) {
      return null;
    }
    String pullOpenTag = "{pull}";
    String openLinkTag = "[#";
    int openTagIndex = description.indexOf(pullOpenTag);
    if (openTagIndex == -1) {
      return null;
    }
    int openlinkTagIndex = description.indexOf(openLinkTag, openTagIndex + pullOpenTag.length());
    if (openlinkTagIndex == -1) {
      System.err.println("Parsing error, found PR open tag at index " + openTagIndex + " but not openLink tag after it in \"" + description + "\"");
      return null;
    }
    return Integer.valueOf(description.substring(openTagIndex + pullOpenTag.length(), openlinkTagIndex));
  }

  /**
   * Extract issue number from strings which looks either like
   * {@code Update the schema for the REST API specification {pull}42346[#42346] (issue: {issue}35262[#35262])}
   * or like
   * {@code Geo: add Geometry-based query builders to QueryBuilders {pull}45058[#45058] (issues: {issue}44715[#44715], {issue}45048[#45048])}
   */
  private int[] parseIssueNumbers(String description) {
    String issuesListOpenTag = "(issues:";
    int issuesListOpenTagIndex = description.indexOf(issuesListOpenTag);
    if (issuesListOpenTagIndex == -1) {
      return parseIssueNumber(description);
    }
    List<Integer> issueNumbers = new ArrayList<>();
    int startIndex = issuesListOpenTagIndex + issuesListOpenTag.length();
    FoundIssueNumber foundIssueNumber;
    while ((foundIssueNumber = findNextIssueNumber(description, startIndex)) != null) {
      issueNumbers.add(foundIssueNumber.issueNumber);
      startIndex = foundIssueNumber.nextStartIndex;
    }
    if (issueNumbers.isEmpty()) {
      return ReleaseNoteDescription.NO_ISSUE_NUMBER;
    }
    return issueNumbers.stream().mapToInt(t -> t).toArray();
  }

  private int[] parseIssueNumber(String description) {
    String issueListOpenTag = "(issue:";
    int issueListOpenTagIndex = description.indexOf(issueListOpenTag);
    if (issueListOpenTagIndex == -1) {
      return ReleaseNoteDescription.NO_ISSUE_NUMBER;
    }
    FoundIssueNumber foundIssueNumber = findNextIssueNumber(description, issueListOpenTagIndex + issueListOpenTag.length());
    if (foundIssueNumber != null) {
      return new int[] {foundIssueNumber.issueNumber};
    }
    return ReleaseNoteDescription.NO_ISSUE_NUMBER;
  }

  // @CheckForNull
  private FoundIssueNumber findNextIssueNumber(String description, int startIndex) {
    String issueOpenTag = "{issue}";
    int issueOpenTagIndex = description.indexOf(issueOpenTag, startIndex);
    if (issueOpenTagIndex == -1) {
      System.err.println("Parsing error, can't found issue open tag after index " + startIndex + " but not issue open tag after it in \"" + description + "\"");
      return null;
    }
    String openLinkTag = "[#";
    int openLinkTagIndex = description.indexOf(openLinkTag, issueOpenTagIndex + issueOpenTag.length());
    if (openLinkTagIndex == -1) {
      System.err.println("Parsing error, found issue open tag at index " + issueOpenTagIndex + " but not openLink tag after it in \"" + description + "\"");
      return null;
    }
    Integer issueNumber = Integer.valueOf(description.substring(issueOpenTagIndex + issueOpenTag.length(), openLinkTagIndex));
    return new FoundIssueNumber(issueNumber, openLinkTagIndex + openLinkTag.length());
  }

  private static final class FoundIssueNumber {
    private final int issueNumber;
    private final int nextStartIndex;

    private FoundIssueNumber(int issueNumber, int nextStartIndex) {
      this.issueNumber = issueNumber;
      this.nextStartIndex = nextStartIndex;
    }
  }

  private static String linkToPullRequest(ReleaseNoteDescription description) {
    Integer prNumber = description.prNumber;
    if (prNumber == null) {
      return "";
    }
    return format("=HYPERLINK(\"\"https://github.com/elastic/elasticsearch/pull/%s\"\",\"\"%s\"\")", prNumber, prNumber);
  }

  private static String linkToFirstIssue(ReleaseNoteDescription description) {
    int[] issueNumbers = description.issueNumbers;
    if (issueNumbers.length == 0) {
      return "";
    }
    return Arrays.stream(issueNumbers)
      .limit(1)
      .mapToObj(ReadEsAsciiDoc::linkToIssue)
      .collect(Collectors.joining(","));
  }

  private static String linkToOtherIssues(ReleaseNoteDescription description) {
    int[] issueNumbers = description.issueNumbers;
    if (issueNumbers.length <= 1) {
      return "";
    }
    return "," + Arrays.stream(issueNumbers)
      .skip(1)
      .mapToObj(issueNumber -> "\"" + linkToIssue(issueNumber) + "\"")
      .collect(Collectors.joining(","));
  }

  private static String linkToIssue(int issueNumber) {
    return format("=HYPERLINK(\"\"https://github.com/elastic/elasticsearch/issues/%s\"\",\"\"%s\"\")", issueNumber, issueNumber);
  }

  private static String googleSheetEscape(String str) {
    if (str.isEmpty() || str.charAt(0) == '=') {
      return str;
    }
    return '\'' + str;
  }

  /**
   * @param path can either be a migration asciidoc file or a directory. If a directory, all files in it will be read and parsed.
   */
  private static Map<String, Map<String, List<MigrationInfo>>> readMigrations(String titlePrefix, String version, Path path) throws IOException {
    MigrationAsciidocReader reader = new MigrationAsciidocReader(titlePrefix, version);
    for (Path file : toFiles(path).toArray(Path[]::new)) {
      Files.lines(file, Charset.defaultCharset()).forEach(reader);
    }
    reader.flush();
    return reader.output;
  }

  private static Stream<Path> toFiles(Path path) {
    if (Files.isRegularFile(path)) {
      return Stream.of(path);
    } else if (Files.isDirectory(path)) {
      Iterable<Path> files = () -> {
        try {
          return Files.newDirectoryStream(path).iterator();
        } catch (IOException e) {
          throw new RuntimeException(e);
        }
      };
      return StreamSupport.stream(files.spliterator(), false);
    } else {
      throw new IllegalArgumentException("Path " + path + " is neither a regular file nor a directory");
    }
  }

  private static final class MigrationInfo {
    private final String title;
    private final String data;

    private MigrationInfo(String title, String data) {
      this.title = title;
      this.data = data;
    }
  }

  private static class MigrationAsciidocReader implements Consumer<String> {
    private final String titlePrefix;
    private final String version;
    private final Map<String, Map<String, List<MigrationInfo>>> output = new LinkedHashMap<>();
    private int lineNumber = 0;
    private String currentCategory;
    private String currentTitle;
    private String currentData;

    private MigrationAsciidocReader(String titlePrefix, String version) {
      this.titlePrefix = titlePrefix;
      this.version = version;
    }

    @Override
    public void accept(String line) {
      lineNumber++;

      if (isAnchorOrFormatting(line) || isComment(line)) {
        return;
      }

      Optional<String> category = readCategory(line);
      if (category.isPresent()) {
        flush();
        currentCategory = category.get();
        currentTitle = null;
        currentData = null;
        return;
      }

      Optional<String> title = readTitle(line);
      if (title.isPresent()) {
        flush();
        currentTitle = title.get();
        currentData = null;
        return;
      }

      if (currentTitle != null && !line.isEmpty() && currentData == null) {
        currentData = line;
        printData();
        return;
      }

      if (currentData != null) {
        currentData += '\n' + line;
        printData();
        return;
      }

      if (!line.isEmpty()) {
        System.err.println(format("??? %s : %s : %s : %s", version, currentCategory, currentTitle, line));
      }
    }

    private void flush() {
      if (currentCategory != null && currentTitle != null) {
        output.compute(version,
          (version, existingMap) -> {
            Map<String, List<MigrationInfo>> value = existingMap == null ? new LinkedHashMap<>() : existingMap;
            value.compute(currentCategory,
              (category, existingList) -> {
                List<MigrationInfo> list = existingList == null ? new ArrayList<>() : existingList;
                list.add(new MigrationInfo(currentTitle, currentData));
                return list;
              });
            return value;
          });
        currentTitle = null;
        currentData = null;
      }
    }

    private void printData() {
      // System.out.println(format("(%s) DATA %s : %s : %s : %s", lineNumber, version, currentCategory, currentTitle, currentData));
    }

    private Optional<String> readCategory(String line) {
      String prefix = titlePrefix + "== ";
      if (line.startsWith(prefix)) {
        return Optional.of(line.substring(prefix.length()));
      }
      return Optional.empty();
    }

    private Optional<String> readTitle(String line) {
      String prefix = titlePrefix + "=== ";
      if (line.startsWith(prefix)) {
        return Optional.of(line.substring(prefix.length()));
      }
      return Optional.empty();
    }
  }

  private static Map<String, Map<ChangeType, List<DataInfo>>> readRelaseNotes(Path... inputs) throws IOException {
    BreakingChangeAsciidocReader reader = new BreakingChangeAsciidocReader();
    for (Path input : inputs) {
      Files.lines(input, Charset.defaultCharset()).forEach(reader);
    }
    reader.flush(false);
    return reader.output;
  }

  private enum ChangeType {
    BREAKING_CHANGES("Breaking Changes"),
    JAVA_BREAKING_CHANGES("Breaking Java Changes"),
    DEPRECATIONS("Deprecations"),
    IGNORED(null);

    private final String text;

    ChangeType(String text) {
      this.text = text;
    }

    public static Optional<ChangeType> fromLine(String line) {
      String prefix = "=== ";
      if (!line.startsWith(prefix)) {
        return Optional.empty();
      }
      Optional<ChangeType> supported = Arrays.stream(values())
        .filter(t -> t.text != null)
        .filter(t -> line.equalsIgnoreCase(prefix + t.text))
        .findFirst();
      if (supported.isPresent()) {
        return supported;
      }
      return Optional.of(IGNORED);
    }
  }

  private static final class DataInfo {
    private final String category;
    private final String info;
    private final String subInfo;

    private DataInfo(String category, String info, String subInfo) {
      this.category = category;
      this.info = info;
      this.subInfo = subInfo;
    }

    @Override
    public String toString() {
      if (subInfo == null) {
        return category + ":: " + info;
      }
      return category + " :: " + info + " : " + subInfo;
    }
  }

  private static class BreakingChangeAsciidocReader implements Consumer<String> {
    private final Map<String, Map<ChangeType, List<DataInfo>>> output = new LinkedHashMap<>();
    private int lineNumber = 0;
    private String currentVersion = null;
    private ChangeType currentChangeType = null;
    private String currentCategory = null;
    private String currentInfo = null;
    private String currentSubInfo = null;

    @Override
    public void accept(String line) {
      lineNumber++;

      if (isAnchorOrFormatting(line)) {
        flush(false);
        return;
      }

      Optional<String> version = readVersion(line);
      if (version.isPresent()) {
        flush(false);
        currentVersion = version.get();
        currentChangeType = null;
        currentCategory = null;
        currentInfo = null;
        currentSubInfo = null;
        return;
      }

      Optional<ChangeType> changeType = currentVersion == null ? Optional.empty() : ChangeType.fromLine(line);
      if (changeType.isPresent()) {
        currentChangeType = changeType.filter(t -> t != ChangeType.IGNORED).orElse(null);
        currentCategory = null;
        currentInfo = null;
        currentSubInfo = null;
        return;
      }

      Optional<String> category = currentChangeType == null ? Optional.empty() : readCategory(line);
      if (category.isPresent()) {
        currentCategory = category.get();
        currentInfo = null;
        currentSubInfo = null;
        return;
      }

      Optional<String> info = currentCategory == null ? Optional.empty() : readInfo(line);
      if (info.isPresent()) {
        flush(false);
        currentInfo = info.get();
        currentSubInfo = null;
        printData();
        return;
      }

      Optional<String> subInfo = currentInfo == null ? Optional.empty() : readSubInfo(line);
      if (subInfo.isPresent()) {
        flush(true);
        currentSubInfo = subInfo.get();
        printData();
        return;
      }

      if (!line.isEmpty()) {
        if (currentSubInfo != null) {
          currentSubInfo += " " + line;
          printData();
          return;
        }
        if (currentInfo != null) {
          currentInfo += " " + line;
          printData();
          return;
        }
      } else {
        flush(false);
      }

      System.err.println(format("??? %s : %s : %s : %s", currentVersion, currentChangeType, currentCategory, line));
    }

    private void flush(boolean onsubInfoOnly) {
      if (currentVersion != null && currentChangeType != null && currentCategory != null && currentInfo != null
        && (!onsubInfoOnly || currentSubInfo != null)) {
        output.compute(currentVersion,
          (version, existingMap) -> {
            Map<ChangeType, List<DataInfo>> value = existingMap == null ? new LinkedHashMap<>() : existingMap;
            value.compute(currentChangeType,
              ((changeType, existingList) -> {
                List<DataInfo> list = existingList == null ? new ArrayList<>() : existingList;
                list.add(new DataInfo(currentCategory, currentInfo, currentSubInfo));
                return list;
              }));
            return value;
          });
        if (!onsubInfoOnly) {
          currentInfo = null;
        }
        currentSubInfo = null;
      }
    }

    private void printData() {
      System.out.println(format("(%s) DATA %s : %s : %s : %s : %s", lineNumber, currentVersion, currentChangeType, currentCategory, currentInfo, currentSubInfo));
    }

    private Optional<String> readSubInfo(String line) {
      if (line.startsWith("** ")) {
        return Optional.of(line.substring("** ".length()));
      }
      return Optional.empty();
    }

    private Optional<String> readInfo(String line) {
      if (line.startsWith("* ")) {
        return Optional.of(line.substring(2));
      }
      return Optional.empty();
    }

    private Optional<String> readVersion(String line) {
      String prefix = "== {es} version ";
      if (line.startsWith(prefix)) {
        return Optional.of(line.substring(prefix.length()));
      }
      return Optional.empty();
    }

    private Optional<String> readCategory(String line) {
      if (line.endsWith("::")) {
        return Optional.of(line.substring(0, line.length() - "::".length()));
      }
      return Optional.empty();
    }
  }

  private static boolean isAnchorOrFormatting(String line) {
    return line.startsWith("[");
  }

  private static boolean isComment(String line) {
    return line.startsWith("// ");
  }
}