Generate backlinks for Bear notes in Scriptable on iOS

Bear.Backlinks.js

// Variables used by Scriptable.
// These must be at the very top of the file. Do not edit.
// icon-color: blue; icon-glyph: link;

/**
 * GENERATE BEAR BACKLINKS
 *
 * This script will find and add backlinks in Bear.
 *
 * !!Please backup your notes before running!! https://bear.app/faq/Backup%20&%20Restore/
 *   I haven't had any issues with running this script, but have only tested it
 *   with my notes. I would strongly suggest that you back up your notes so you
 *   can restore them if you don't like the outcome.
 *
 * INSTRUCTIONS
 *    1. Edit the Settings below this comment block
 *    2. Turn on "Reduce motion" setting: https://support.apple.com/en-gb/HT202655
 *      - This isn't mandatory, but will speed up the execution of the script. Because
 *        we have to make a roundtrip between Scriptable and Bear for each note evaluated,
 *        this can take a very long time to run (I've had it run for ~30 minutes with 770 notes).
 *        Turning reduce motion on significantly reduces the amount of time each roundtrip takes.
 *      - [UPDATE 2020-11-11 -- the script seems to be broken in Split View, probably due to some OS or app changes]
 *        -If you run this on an iPad with split view support, having Scriptble and Bear open
 *        next to each other makes this run exponentially faster, as there is no app switching.-
 *    3. Run script
 *      - NB! You are effectively locked out of your device while this is running. You can quit
 *        the apps if you're fast enough, but it is challenging. Make sure you won't need the device
 *        while this is running.
 */

//
// SETTINGS
//

// The results of this search will be the body of notes used to find backlinks.
// The default here shows all notes that aren't locked (which for me is all notes).
// The search term can be tested in Bear to see which notes will be included.
// https://bear.app/faq/Advanced%20search%20options%20in%20Bear/
const NOTES_SEARCH_TERM = "-@locked";

/**
 * Place token for your device between quotes below. Note that different devices have different tokens.
 * If you use this script on different devices, you can use Device.isPad(), for example, to choose the right one.
 *
 * From Bear documentation (https://bear.app/faq/X-callback-url%20Scheme%20documentation/):
 *
 * In order to extend their functionalties, some of the API calls allow an app generated token to be
 * passed along with the other parameters. Please mind a Token generated on iOS is not valid for MacOS and vice-versa.
 *
 * On MacOS, select Help → API Token → Copy Token and will be available in your pasteboard.
 *
 * On iOS go to the preferences → General, locate the API Token section and tap the cell below
 * to generate the token or copy it in your pasteboard.
 */
const BEAR_TOKEN = "";

//
//
//
//
//
//
//
//
//

//
// HELPERS
//

const uniqueArray = (...arrays) => [...new Set(arrays.flatMap(i => i))];

const noteLinkInNoteRegex = /\[\[(.+?)\]\]/g;
/** @param {string} noteBody */
const getNoteLinks = noteBody =>
  uniqueArray(
    noteBody
      .split("\n")
      .flatMap(line =>
        [...line.matchAll(noteLinkInNoteRegex)].map(match => match[1])
      )
      .filter(Boolean)
  );

/** Do string arrays have same values, in any order? */
const stringArraysHaveSameValues = (arr1, arr2) => {
  if (arr1.length !== arr2.length) return false;
  return arr1.every(arr1Val => arr2.some(arr2Val => arr1Val === arr2Val));
};

/**
 * Given array of strings, return array of lines removing all empty lines
 * at beginning and end of lines.
 */
const trimLines = lines => {
  const { firstContentLine, lastContentLine } = lines.reduce(
    (acc, line, i) => {
      const lineHasContent = Boolean(line.trim().length);
      if (acc.firstContentLine === -1 && lineHasContent)
        acc.firstContentLine = i;
      if (lineHasContent) acc.lastContentLine = i;
      return acc;
    },
    { firstContentLine: -1, lastContentLine: -1 }
  );
  return lastContentLine === -1
    ? []
    : lines.slice(firstContentLine, lastContentLine + 1);
};

//
// BEAR XCALLBACK FUNCTIONS
//

const BASE_URL = "bear://x-callback-url";
const getBearCallbackObject = (endpoint, params) => {
  const callbackObject = new CallbackURL(`${BASE_URL}/${endpoint}`);
  Object.entries(params).forEach(([key, val]) =>
    callbackObject.addParameter(key, val)
  );
  callbackObject.addParameter("token", BEAR_TOKEN);
  return callbackObject;
};

const getFullBearNote = async noteId => {
  const callback = getBearCallbackObject("open-note", {
    id: noteId,
    open_note: "no",
  });
  return await callback.open();
};

const getBearSearchResults = async term => {
  const callback = getBearCallbackObject("search", { term });
  const resultsRaw = await callback.open();
  return JSON.parse(resultsRaw.notes);
};

const replaceBearNoteBody = async (noteId, newNoteBody) => {
  const callback = getBearCallbackObject("add-text", {
    id: noteId,
    text: newNoteBody,
    mode: "replace_all",
    open_note: "no",
  });
  return await callback.open();
};

//
// NOTE PARSING
//

const METADATA_DIVIDER = "---";
const METADATA_TITLE = "::*METADATA*::";
const METADATA_LINE_PREFIX = "\t- [[";

/** With ability to link to sections in notes, notes w/ "/" in title must get special handling */
const cleanNoteLink = link => {
  const hasSectionLink = /[^\\](\/.+$)/.test(link);
  const withoutSectionLink = (() => {
    if (!hasSectionLink) return link;
    const splitBySlashes = link.split("/");
    splitBySlashes.pop();
    return splitBySlashes.join("/");
  })();
  return withoutSectionLink.replace(/\\\//g, "/");
};

const getMetadataFromNote = note => {
  const defaultReturn = {
    noteWithoutMetadata: note,
    currentBacklinks: [],
  };
  const lines = note.split("\n");
  const metadataTitleLineIndex = lines.indexOf(METADATA_TITLE);
  if (metadataTitleLineIndex === -1) return defaultReturn;
  const closingDividerIndex = lines.findIndex(
    (line, i) => i > metadataTitleLineIndex && line === METADATA_DIVIDER
  );
  if (closingDividerIndex === -1) return defaultReturn;
  const metadataLines = lines.splice(
    metadataTitleLineIndex - 1,
    closingDividerIndex - (metadataTitleLineIndex - 1) + 1
  );
  const currentBacklinks = metadataLines
    .filter(line => line.startsWith(METADATA_LINE_PREFIX))
    .map(line => line.replace(METADATA_LINE_PREFIX, "").replace("]]", ""))
    .map(cleanNoteLink);
  return { noteWithoutMetadata: lines.join("\n"), currentBacklinks };
};

const getForwardLinks = (noteWithoutMetadata, noteTitle) =>
  getNoteLinks(noteWithoutMetadata)
    .map(cleanNoteLink)
    // This can happen if linking to a subsection within a note
    .filter(forwardLink => forwardLink !== noteTitle);

/**
 * To start, get full notebody for all links that may have note links in them.
 * False positives are removed, leaving a cache of notes that link to other notes.
 * False positive note titles are logged in console; correcting this (they contain "[[")
 * can speed up the script, especially on iPhone.
 */
const populateCacheWithNotesWithLinks = async () => {
  const allNotesThatMayHaveNoteLinks = await getBearSearchResults("[[");
  return (
    await Promise.all(
      allNotesThatMayHaveNoteLinks.map(async ({ identifier, title }) => {
        const { note } = await getFullBearNote(identifier);
        const { noteWithoutMetadata, currentBacklinks } = getMetadataFromNote(
          note
        );
        const forwardLinksInBody = getForwardLinks(noteWithoutMetadata, title);
        const isFalsePositive = !(
          currentBacklinks.length || forwardLinksInBody.length
        );
        if (isFalsePositive) {
          console.log(
            `Note "${title}" matches search "[[", but contains no note links.`
          );
          return null;
        }
        return {
          identifier,
          title,
          noteWithoutMetadata,
          forwardLinksInBody,
          currentBacklinks,
        };
      })
    )
  ).filter(Boolean);
};

/**
 * Initial cache load only pulls from notes that contain "[[",
 * so some target notes without links in them may be missing.
 */
const completeCacheWithNotesWithoutLinks = async cache => {
  const allNotes = await getBearSearchResults(NOTES_SEARCH_TERM);
  const allLinkedNoteTitles = uniqueArray(
    cache.flatMap(({ currentBacklinks, forwardLinksInBody }) => [
      ...currentBacklinks,
      ...forwardLinksInBody,
    ])
  );
  const linkedNotesNotInCache = allLinkedNoteTitles
    .filter(
      noteTitle => !cache.some(cachedNote => cachedNote.title === noteTitle)
    )
    .map(linkedNoteTitle =>
      allNotes.find(note => note.title === linkedNoteTitle)
    )
    .filter(Boolean);
  await Promise.all(
    linkedNotesNotInCache.map(async ({ identifier, title }) => {
      const { note } = await getFullBearNote(identifier);
      cache.push({
        identifier,
        title,
        noteWithoutMetadata: note,
        forwardLinksInBody: [],
        currentBacklinks: [],
      });
    })
  );
};

/** Re-organize cache data into pairs of link target note ID & array of source titles linking to it. */
const getBacklinkIndex = cache => {
  const allForwardLinks = uniqueArray(
    cache.flatMap(({ forwardLinksInBody }) => forwardLinksInBody)
  );
  return allForwardLinks.map(targetNoteTitle => ({
    targetNoteTitle,
    linkSourceTitles: cache
      .filter(({ forwardLinksInBody }) =>
        forwardLinksInBody.includes(targetNoteTitle)
      )
      .map(({ title }) => title),
  }));
};

const getMetadataLines = backlinkTitles =>
  backlinkTitles.length
    ? [
        METADATA_DIVIDER,
        METADATA_TITLE,
        "### Backlinks",
        backlinkTitles
          // Must escape the slash per Bear linking mechanics
          .map(title => `\t- [[${title.replace(/\//g, "\\/")}]]`)
          .join("\n"),
        METADATA_DIVIDER,
      ]
    : null;

/**
 * If the cached note has backlinks, they are different from those in backlinkIndex,
 * the backlinks should be updated.
 */
const hasOutdatedBacklinks = ({ title, currentBacklinks }, backlinkIndex) =>
  backlinkIndex.some(
    ({ targetNoteTitle, linkSourceTitles }) =>
      targetNoteTitle === title &&
      !stringArraysHaveSameValues(linkSourceTitles, currentBacklinks)
  );

/** If the cached note has backlinks, but no entry in backlinkIndex, the backlinks are no longer valid. */
const areAllBacklinksMissing = ({ title, currentBacklinks }, backlinkIndex) =>
  currentBacklinks.length &&
  !backlinkIndex.some(({ targetNoteTitle }) => title === targetNoteTitle);

/** Returns cached note + backlink metadata for notes in cache that need to be updated. */
const getNotesChangesToPush = (cache, backlinkIndex) =>
  cache
    .filter(
      cachedNote =>
        hasOutdatedBacklinks(cachedNote, backlinkIndex) ||
        areAllBacklinksMissing(cachedNote, backlinkIndex)
    )
    .map(({ title, identifier, noteWithoutMetadata }) => {
      const backlinkIndexData = backlinkIndex.find(
        ({ targetNoteTitle }) => title === targetNoteTitle
      );
      if (!backlinkIndexData) return null;
      const { linkSourceTitles } = backlinkIndexData;
      return { identifier, noteWithoutMetadata, linkSourceTitles };
    })
    .filter(Boolean);

const createBacklinks = async () => {
  const cachedNotes = await populateCacheWithNotesWithLinks();
  await completeCacheWithNotesWithoutLinks(cachedNotes);
  const backlinkIndex = getBacklinkIndex(cachedNotes);
  const numResults = (
    await Promise.all(
      getNotesChangesToPush(cachedNotes, backlinkIndex).map(
        async ({ linkSourceTitles, identifier, noteWithoutMetadata }) =>
          await replaceBearNoteBody(
            identifier,
            [
              ...trimLines(noteWithoutMetadata.split("\n")),
              ...(getMetadataLines(linkSourceTitles) || []),
            ].join("\n")
          )
      )
    )
  ).length;
  console.log(`Backlink parsing done -- updated ${numResults} notes.`);
};

await createBacklinks();

@johnchandler 👋 Testing this out now, I also have this issue. I'll report back. Right now it seems to be something to do with having both apps open in split view, but I'll test a few permutations. Not sure if this is due to Scriptable or iOS changes.

Edit: yeah it seems to be only an issue with having both apps open in split view. It works fine for me when switching back and forth between the apps in full screen. Really disappointing since split view sped it up so much. I don’t think there’s anything I can do about this and as I said not sure if it’s caused by an app or OS change

@jsloat This is awesome! Does it work with Wiki links with note headers?

E.g. [[Page Title/Header Title]]

https://blog.bear.app/2020/09/bear-1-7-15-is-out-with-new-features-for-wiki-links-html-export-and-more/

If you change the regex from /\[\[(.+?)\]\]/g to /\[\[([\w\d\s]+)(?:\/.+|)\]\]/g on line 75 it will create backlinks for wiki links with headers references...assuming the intended behavior is to create backlinks for individual notes by ignoring the header reference.

@xlZeroAccesslx nice catch 👍 I have actually updated my own code to support this but didn't update here. I think your solution would probably work but I will just update the whole file with my latest version, will be up within a half hour

Updated version posted -- I refactored this for my devices a couple months ago so the code will look different, but the basic functionality is the same and it's backwards compatible if you've already been using it. But per usual, probably a good idea to backup your Bear database before trying :)

Only thing that was removed was the "blacklist" notes (array of note titles to exclude as link sources). I wasn't using this so I removed, but if anyone misses it I can add it back in.

The new version supports wiki links/note headers as @xlZeroAccesslx mentions above, plus some smaller tweaks that aren't really noticeable. I think this version uses less device memory so if anyone was experiencing slowness/crashes before, this may fix that.

Great work man!

I would love if I could see some context in the backlinks as well, pulling in the paragraph were the link is mentioned.

Here is my backlinks from a page like the script is working now:

epigenetics

::METADATA::

Backlinks

• [[S- 2020 - Who We Are - YouTube - Documentary]]

---

Here is how I ideally want it to look to get more context:

epigenetics

::METADATA::

Backlinks

• [[S- 2020 - Who We Are - YouTube - Documentary]]
  • [[epigenetics]] - Control above the genes.

@Torgithub thanks! This is a nice idea, definitely possible. I may give this a go at some point, but probably not in the near-term.