Generate backlinks for Bear notes in Scriptable on iOS

Bear.Backlinks.js

// Variables used by Scriptable.
// These must be at the very top of the file. Do not edit.
// icon-color: blue; icon-glyph: link;

/**
 * GENERATE BEAR BACKLINKS
 *
 * This script will find and add backlinks in Bear.
 *
 * !!Please backup your notes before running!! https://bear.app/faq/Backup%20&%20Restore/
 *   I haven't had any issues with running this script, but have only tested it
 *   with my notes. I would strongly suggest that you back up your notes so you
 *   can restore them if you don't like the outcome.
 *
 * INSTRUCTIONS
 *    1. Edit the Settings below this comment block
 *    2. Turn on "Reduce motion" setting: https://support.apple.com/en-gb/HT202655
 *      - This isn't mandatory, but will speed up the execution of the script. Because
 *        we have to make a roundtrip between Scriptable and Bear for each note evaluated,
 *        this can take a very long time to run (I've had it run for ~30 minutes with 770 notes).
 *        Turning reduce motion on significantly reduces the amount of time each roundtrip takes.
 *      - [UPDATE 2020-11-11 -- the script seems to be broken in Split View, probably due to some OS or app changes]
 *        -If you run this on an iPad with split view support, having Scriptble and Bear open
 *        next to each other makes this run exponentially faster, as there is no app switching.-
 *    3. Run script
 *      - NB! You are effectively locked out of your device while this is running. You can quit
 *        the apps if you're fast enough, but it is challenging. Make sure you won't need the device
 *        while this is running.
 */

//
// SETTINGS
//

// The results of this search will be the body of notes used to find backlinks.
// The default here shows all notes that aren't locked (which for me is all notes).
// The search term can be tested in Bear to see which notes will be included.
// https://bear.app/faq/Advanced%20search%20options%20in%20Bear/
const NOTES_SEARCH_TERM = "-@locked";

/**
 * Place token for your device between quotes below. Note that different devices have different tokens.
 * If you use this script on different devices, you can use Device.isPad(), for example, to choose the right one.
 *
 * From Bear documentation (https://bear.app/faq/X-callback-url%20Scheme%20documentation/):
 *
 * In order to extend their functionalties, some of the API calls allow an app generated token to be
 * passed along with the other parameters. Please mind a Token generated on iOS is not valid for MacOS and vice-versa.
 *
 * On MacOS, select Help → API Token → Copy Token and will be available in your pasteboard.
 *
 * On iOS go to the preferences → General, locate the API Token section and tap the cell below
 * to generate the token or copy it in your pasteboard.
 */
const BEAR_TOKEN = "";

//
//
//
//
//
//
//
//
//

//
// HELPERS
//

const uniqueArray = (...arrays) => [...new Set(arrays.flatMap(i => i))];

const noteLinkInNoteRegex = /\[\[(.+?)\]\]/g;
/** @param {string} noteBody */
const getNoteLinks = noteBody =>
  uniqueArray(
    noteBody
      .split("\n")
      .flatMap(line =>
        [...line.matchAll(noteLinkInNoteRegex)].map(match => match[1])
      )
      .filter(Boolean)
  );

/** Do string arrays have same values, in any order? */
const stringArraysHaveSameValues = (arr1, arr2) => {
  if (arr1.length !== arr2.length) return false;
  return arr1.every(arr1Val => arr2.some(arr2Val => arr1Val === arr2Val));
};

/**
 * Given array of strings, return array of lines removing all empty lines
 * at beginning and end of lines.
 */
const trimLines = lines => {
  const { firstContentLine, lastContentLine } = lines.reduce(
    (acc, line, i) => {
      const lineHasContent = Boolean(line.trim().length);
      if (acc.firstContentLine === -1 && lineHasContent)
        acc.firstContentLine = i;
      if (lineHasContent) acc.lastContentLine = i;
      return acc;
    },
    { firstContentLine: -1, lastContentLine: -1 }
  );
  return lastContentLine === -1
    ? []
    : lines.slice(firstContentLine, lastContentLine + 1);
};

//
// BEAR XCALLBACK FUNCTIONS
//

const BASE_URL = "bear://x-callback-url";
const getBearCallbackObject = (endpoint, params) => {
  const callbackObject = new CallbackURL(`${BASE_URL}/${endpoint}`);
  Object.entries(params).forEach(([key, val]) =>
    callbackObject.addParameter(key, val)
  );
  callbackObject.addParameter("token", BEAR_TOKEN);
  return callbackObject;
};

const getFullBearNote = async noteId => {
  const callback = getBearCallbackObject("open-note", {
    id: noteId,
    open_note: "no",
  });
  return await callback.open();
};

const getBearSearchResults = async term => {
  const callback = getBearCallbackObject("search", { term });
  const resultsRaw = await callback.open();
  return JSON.parse(resultsRaw.notes);
};

const replaceBearNoteBody = async (noteId, newNoteBody) => {
  const callback = getBearCallbackObject("add-text", {
    id: noteId,
    text: newNoteBody,
    mode: "replace_all",
    open_note: "no",
  });
  return await callback.open();
};

//
// NOTE PARSING
//

const METADATA_DIVIDER = "---";
const METADATA_TITLE = "::*METADATA*::";
const METADATA_LINE_PREFIX = "\t- [[";

/** With ability to link to sections in notes, notes w/ "/" in title must get special handling */
const cleanNoteLink = link => {
  const hasSectionLink = /[^\\](\/.+$)/.test(link);
  const withoutSectionLink = (() => {
    if (!hasSectionLink) return link;
    const splitBySlashes = link.split("/");
    splitBySlashes.pop();
    return splitBySlashes.join("/");
  })();
  return withoutSectionLink.replace(/\\\//g, "/");
};

const getMetadataFromNote = note => {
  const defaultReturn = {
    noteWithoutMetadata: note,
    currentBacklinks: [],
  };
  const lines = note.split("\n");
  const metadataTitleLineIndex = lines.indexOf(METADATA_TITLE);
  if (metadataTitleLineIndex === -1) return defaultReturn;
  const closingDividerIndex = lines.findIndex(
    (line, i) => i > metadataTitleLineIndex && line === METADATA_DIVIDER
  );
  if (closingDividerIndex === -1) return defaultReturn;
  const metadataLines = lines.splice(
    metadataTitleLineIndex - 1,
    closingDividerIndex - (metadataTitleLineIndex - 1) + 1
  );
  const currentBacklinks = metadataLines
    .filter(line => line.startsWith(METADATA_LINE_PREFIX))
    .map(line => line.replace(METADATA_LINE_PREFIX, "").replace("]]", ""))
    .map(cleanNoteLink);
  return { noteWithoutMetadata: lines.join("\n"), currentBacklinks };
};

const getForwardLinks = (noteWithoutMetadata, noteTitle) =>
  getNoteLinks(noteWithoutMetadata)
    .map(cleanNoteLink)
    // This can happen if linking to a subsection within a note
    .filter(forwardLink => forwardLink !== noteTitle);

/**
 * To start, get full notebody for all links that may have note links in them.
 * False positives are removed, leaving a cache of notes that link to other notes.
 * False positive note titles are logged in console; correcting this (they contain "[[")
 * can speed up the script, especially on iPhone.
 */
const populateCacheWithNotesWithLinks = async () => {
  const allNotesThatMayHaveNoteLinks = await getBearSearchResults("[[");
  return (
    await Promise.all(
      allNotesThatMayHaveNoteLinks.map(async ({ identifier, title }) => {
        const { note } = await getFullBearNote(identifier);
        const { noteWithoutMetadata, currentBacklinks } = getMetadataFromNote(
          note
        );
        const forwardLinksInBody = getForwardLinks(noteWithoutMetadata, title);
        const isFalsePositive = !(
          currentBacklinks.length || forwardLinksInBody.length
        );
        if (isFalsePositive) {
          console.log(
            `Note "${title}" matches search "[[", but contains no note links.`
          );
          return null;
        }
        return {
          identifier,
          title,
          noteWithoutMetadata,
          forwardLinksInBody,
          currentBacklinks,
        };
      })
    )
  ).filter(Boolean);
};

/**
 * Initial cache load only pulls from notes that contain "[[",
 * so some target notes without links in them may be missing.
 */
const completeCacheWithNotesWithoutLinks = async cache => {
  const allNotes = await getBearSearchResults(NOTES_SEARCH_TERM);
  const allLinkedNoteTitles = uniqueArray(
    cache.flatMap(({ currentBacklinks, forwardLinksInBody }) => [
      ...currentBacklinks,
      ...forwardLinksInBody,
    ])
  );
  const linkedNotesNotInCache = allLinkedNoteTitles
    .filter(
      noteTitle => !cache.some(cachedNote => cachedNote.title === noteTitle)
    )
    .map(linkedNoteTitle =>
      allNotes.find(note => note.title === linkedNoteTitle)
    )
    .filter(Boolean);
  await Promise.all(
    linkedNotesNotInCache.map(async ({ identifier, title }) => {
      const { note } = await getFullBearNote(identifier);
      cache.push({
        identifier,
        title,
        noteWithoutMetadata: note,
        forwardLinksInBody: [],
        currentBacklinks: [],
      });
    })
  );
};

/** Re-organize cache data into pairs of link target note ID & array of source titles linking to it. */
const getBacklinkIndex = cache => {
  const allForwardLinks = uniqueArray(
    cache.flatMap(({ forwardLinksInBody }) => forwardLinksInBody)
  );
  return allForwardLinks.map(targetNoteTitle => ({
    targetNoteTitle,
    linkSourceTitles: cache
      .filter(({ forwardLinksInBody }) =>
        forwardLinksInBody.includes(targetNoteTitle)
      )
      .map(({ title }) => title),
  }));
};

const getMetadataLines = backlinkTitles =>
  backlinkTitles.length
    ? [
        METADATA_DIVIDER,
        METADATA_TITLE,
        "### Backlinks",
        backlinkTitles
          // Must escape the slash per Bear linking mechanics
          .map(title => `\t- [[${title.replace(/\//g, "\\/")}]]`)
          .join("\n"),
        METADATA_DIVIDER,
      ]
    : null;

/**
 * If the cached note has backlinks, they are different from those in backlinkIndex,
 * the backlinks should be updated.
 */
const hasOutdatedBacklinks = ({ title, currentBacklinks }, backlinkIndex) =>
  backlinkIndex.some(
    ({ targetNoteTitle, linkSourceTitles }) =>
      targetNoteTitle === title &&
      !stringArraysHaveSameValues(linkSourceTitles, currentBacklinks)
  );

/** If the cached note has backlinks, but no entry in backlinkIndex, the backlinks are no longer valid. */
const areAllBacklinksMissing = ({ title, currentBacklinks }, backlinkIndex) =>
  currentBacklinks.length &&
  !backlinkIndex.some(({ targetNoteTitle }) => title === targetNoteTitle);

/** Returns cached note + backlink metadata for notes in cache that need to be updated. */
const getNotesChangesToPush = (cache, backlinkIndex) =>
  cache
    .filter(
      cachedNote =>
        hasOutdatedBacklinks(cachedNote, backlinkIndex) ||
        areAllBacklinksMissing(cachedNote, backlinkIndex)
    )
    .map(({ title, identifier, noteWithoutMetadata }) => {
      const backlinkIndexData = backlinkIndex.find(
        ({ targetNoteTitle }) => title === targetNoteTitle
      );
      if (!backlinkIndexData) return null;
      const { linkSourceTitles } = backlinkIndexData;
      return { identifier, noteWithoutMetadata, linkSourceTitles };
    })
    .filter(Boolean);

const createBacklinks = async () => {
  const cachedNotes = await populateCacheWithNotesWithLinks();
  await completeCacheWithNotesWithoutLinks(cachedNotes);
  const backlinkIndex = getBacklinkIndex(cachedNotes);
  const numResults = (
    await Promise.all(
      getNotesChangesToPush(cachedNotes, backlinkIndex).map(
        async ({ linkSourceTitles, identifier, noteWithoutMetadata }) =>
          await replaceBearNoteBody(
            identifier,
            [
              ...trimLines(noteWithoutMetadata.split("\n")),
              ...(getMetadataLines(linkSourceTitles) || []),
            ].join("\n")
          )
      )
    )
  ).length;
  console.log(`Backlink parsing done -- updated ${numResults} notes.`);
};

await createBacklinks();

@jsloat This is awesome! Does it work with Wiki links with note headers?

E.g. [[Page Title/Header Title]]

https://blog.bear.app/2020/09/bear-1-7-15-is-out-with-new-features-for-wiki-links-html-export-and-more/

If you change the regex from /\[\[(.+?)\]\]/g to /\[\[([\w\d\s]+)(?:\/.+|)\]\]/g on line 75 it will create backlinks for wiki links with headers references...assuming the intended behavior is to create backlinks for individual notes by ignoring the header reference.

@xlZeroAccesslx nice catch 👍 I have actually updated my own code to support this but didn't update here. I think your solution would probably work but I will just update the whole file with my latest version, will be up within a half hour

Updated version posted -- I refactored this for my devices a couple months ago so the code will look different, but the basic functionality is the same and it's backwards compatible if you've already been using it. But per usual, probably a good idea to backup your Bear database before trying :)

Only thing that was removed was the "blacklist" notes (array of note titles to exclude as link sources). I wasn't using this so I removed, but if anyone misses it I can add it back in.

The new version supports wiki links/note headers as @xlZeroAccesslx mentions above, plus some smaller tweaks that aren't really noticeable. I think this version uses less device memory so if anyone was experiencing slowness/crashes before, this may fix that.

Great work man!

I would love if I could see some context in the backlinks as well, pulling in the paragraph were the link is mentioned.

Here is my backlinks from a page like the script is working now:

epigenetics

::METADATA::

Backlinks

• [[S- 2020 - Who We Are - YouTube - Documentary]]

---

Here is how I ideally want it to look to get more context:

epigenetics

::METADATA::

Backlinks

• [[S- 2020 - Who We Are - YouTube - Documentary]]
  • [[epigenetics]] - Control above the genes.

@Torgithub thanks! This is a nice idea, definitely possible. I may give this a go at some point, but probably not in the near-term.