jsloat/Bear.Backlinks.js

## Bear.Backlinks.js
// Variables used by Scriptable.
// These must be at the very top of the file. Do not edit.
// icon-color: blue; icon-glyph: link;

/**
 * GENERATE BEAR BACKLINKS
 *
 * This script will find and add backlinks in Bear.
 *
 * !!Please backup your notes before running!! https://bear.app/faq/Backup%20&%20Restore/
 *   I haven't had any issues with running this script, but have only tested it
 *   with my notes. I would strongly suggest that you back up your notes so you
 *   can restore them if you don't like the outcome.
 *
 * INSTRUCTIONS
 *    1. Edit the Settings below this comment block
 *    2. Turn on "Reduce motion" setting: https://support.apple.com/en-gb/HT202655
 *      - This isn't mandatory, but will speed up the execution of the script. Because
 *        we have to make a roundtrip between Scriptable and Bear for each note evaluated,
 *        this can take a very long time to run (I've had it run for ~30 minutes with 770 notes).
 *        Turning reduce motion on significantly reduces the amount of time each roundtrip takes.
 *      - [UPDATE 2020-11-11 -- the script seems to be broken in Split View, probably due to some OS or app changes]
 *        -If you run this on an iPad with split view support, having Scriptble and Bear open
 *        next to each other makes this run exponentially faster, as there is no app switching.-
 *    3. Run script
 *      - NB! You are effectively locked out of your device while this is running. You can quit
 *        the apps if you're fast enough, but it is challenging. Make sure you won't need the device
 *        while this is running.
 */

//
// SETTINGS
//

// The results of this search will be the body of notes used to find backlinks.
// The default here shows all notes that aren't locked (which for me is all notes).
// The search term can be tested in Bear to see which notes will be included.
// https://bear.app/faq/Advanced%20search%20options%20in%20Bear/
const NOTES_SEARCH_TERM = "-@locked";

/**
 * Place token for your device between quotes below. Note that different devices have different tokens.
 * If you use this script on different devices, you can use Device.isPad(), for example, to choose the right one.
 *
 * From Bear documentation (https://bear.app/faq/X-callback-url%20Scheme%20documentation/):
 *
 * In order to extend their functionalties, some of the API calls allow an app generated token to be
 * passed along with the other parameters. Please mind a Token generated on iOS is not valid for MacOS and vice-versa.
 *
 * On MacOS, select Help → API Token → Copy Token and will be available in your pasteboard.
 *
 * On iOS go to the preferences → General, locate the API Token section and tap the cell below
 * to generate the token or copy it in your pasteboard.
 */
const BEAR_TOKEN = "";

//
//
//
//
//
//
//
//
//

//
// HELPERS
//

const uniqueArray = (...arrays) => [...new Set(arrays.flatMap(i => i))];

const noteLinkInNoteRegex = /\[\[(.+?)\]\]/g;
/** @param {string} noteBody */
const getNoteLinks = noteBody =>
  uniqueArray(
    noteBody
      .split("\n")
      .flatMap(line =>
        [...line.matchAll(noteLinkInNoteRegex)].map(match => match[1])
      )
      .filter(Boolean)
  );

/** Do string arrays have same values, in any order? */
const stringArraysHaveSameValues = (arr1, arr2) => {
  if (arr1.length !== arr2.length) return false;
  return arr1.every(arr1Val => arr2.some(arr2Val => arr1Val === arr2Val));
};

/**
 * Given array of strings, return array of lines removing all empty lines
 * at beginning and end of lines.
 */
const trimLines = lines => {
  const { firstContentLine, lastContentLine } = lines.reduce(
    (acc, line, i) => {
      const lineHasContent = Boolean(line.trim().length);
      if (acc.firstContentLine === -1 && lineHasContent)
        acc.firstContentLine = i;
      if (lineHasContent) acc.lastContentLine = i;
      return acc;
    },
    { firstContentLine: -1, lastContentLine: -1 }
  );
  return lastContentLine === -1
    ? []
    : lines.slice(firstContentLine, lastContentLine + 1);
};

//
// BEAR XCALLBACK FUNCTIONS
//

const BASE_URL = "bear://x-callback-url";
const getBearCallbackObject = (endpoint, params) => {
  const callbackObject = new CallbackURL(`${BASE_URL}/${endpoint}`);
  Object.entries(params).forEach(([key, val]) =>
    callbackObject.addParameter(key, val)
  );
  callbackObject.addParameter("token", BEAR_TOKEN);
  return callbackObject;
};

const getFullBearNote = async noteId => {
  const callback = getBearCallbackObject("open-note", {
    id: noteId,
    open_note: "no",
  });
  return await callback.open();
};

const getBearSearchResults = async term => {
  const callback = getBearCallbackObject("search", { term });
  const resultsRaw = await callback.open();
  return JSON.parse(resultsRaw.notes);
};

const replaceBearNoteBody = async (noteId, newNoteBody) => {
  const callback = getBearCallbackObject("add-text", {
    id: noteId,
    text: newNoteBody,
    mode: "replace_all",
    open_note: "no",
  });
  return await callback.open();
};

//
// NOTE PARSING
//

const METADATA_DIVIDER = "---";
const METADATA_TITLE = "::*METADATA*::";
const METADATA_LINE_PREFIX = "\t- [[";

/** With ability to link to sections in notes, notes w/ "/" in title must get special handling */
const cleanNoteLink = link => {
  const hasSectionLink = /[^\\](\/.+$)/.test(link);
  const withoutSectionLink = (() => {
    if (!hasSectionLink) return link;
    const splitBySlashes = link.split("/");
    splitBySlashes.pop();
    return splitBySlashes.join("/");
  })();
  return withoutSectionLink.replace(/\\\//g, "/");
};

const getMetadataFromNote = note => {
  const defaultReturn = {
    noteWithoutMetadata: note,
    currentBacklinks: [],
  };
  const lines = note.split("\n");
  const metadataTitleLineIndex = lines.indexOf(METADATA_TITLE);
  if (metadataTitleLineIndex === -1) return defaultReturn;
  const closingDividerIndex = lines.findIndex(
    (line, i) => i > metadataTitleLineIndex && line === METADATA_DIVIDER
  );
  if (closingDividerIndex === -1) return defaultReturn;
  const metadataLines = lines.splice(
    metadataTitleLineIndex - 1,
    closingDividerIndex - (metadataTitleLineIndex - 1) + 1
  );
  const currentBacklinks = metadataLines
    .filter(line => line.startsWith(METADATA_LINE_PREFIX))
    .map(line => line.replace(METADATA_LINE_PREFIX, "").replace("]]", ""))
    .map(cleanNoteLink);
  return { noteWithoutMetadata: lines.join("\n"), currentBacklinks };
};

const getForwardLinks = (noteWithoutMetadata, noteTitle) =>
  getNoteLinks(noteWithoutMetadata)
    .map(cleanNoteLink)
    // This can happen if linking to a subsection within a note
    .filter(forwardLink => forwardLink !== noteTitle);

/**
 * To start, get full notebody for all links that may have note links in them.
 * False positives are removed, leaving a cache of notes that link to other notes.
 * False positive note titles are logged in console; correcting this (they contain "[[")
 * can speed up the script, especially on iPhone.
 */
const populateCacheWithNotesWithLinks = async () => {
  const allNotesThatMayHaveNoteLinks = await getBearSearchResults("[[");
  return (
    await Promise.all(
      allNotesThatMayHaveNoteLinks.map(async ({ identifier, title }) => {
        const { note } = await getFullBearNote(identifier);
        const { noteWithoutMetadata, currentBacklinks } = getMetadataFromNote(
          note
        );
        const forwardLinksInBody = getForwardLinks(noteWithoutMetadata, title);
        const isFalsePositive = !(
          currentBacklinks.length || forwardLinksInBody.length
        );
        if (isFalsePositive) {
          console.log(
            `Note "${title}" matches search "[[", but contains no note links.`
          );
          return null;
        }
        return {
          identifier,
          title,
          noteWithoutMetadata,
          forwardLinksInBody,
          currentBacklinks,
        };
      })
    )
  ).filter(Boolean);
};

/**
 * Initial cache load only pulls from notes that contain "[[",
 * so some target notes without links in them may be missing.
 */
const completeCacheWithNotesWithoutLinks = async cache => {
  const allNotes = await getBearSearchResults(NOTES_SEARCH_TERM);
  const allLinkedNoteTitles = uniqueArray(
    cache.flatMap(({ currentBacklinks, forwardLinksInBody }) => [
      ...currentBacklinks,
      ...forwardLinksInBody,
    ])
  );
  const linkedNotesNotInCache = allLinkedNoteTitles
    .filter(
      noteTitle => !cache.some(cachedNote => cachedNote.title === noteTitle)
    )
    .map(linkedNoteTitle =>
      allNotes.find(note => note.title === linkedNoteTitle)
    )
    .filter(Boolean);
  await Promise.all(
    linkedNotesNotInCache.map(async ({ identifier, title }) => {
      const { note } = await getFullBearNote(identifier);
      cache.push({
        identifier,
        title,
        noteWithoutMetadata: note,
        forwardLinksInBody: [],
        currentBacklinks: [],
      });
    })
  );
};

/** Re-organize cache data into pairs of link target note ID & array of source titles linking to it. */
const getBacklinkIndex = cache => {
  const allForwardLinks = uniqueArray(
    cache.flatMap(({ forwardLinksInBody }) => forwardLinksInBody)
  );
  return allForwardLinks.map(targetNoteTitle => ({
    targetNoteTitle,
    linkSourceTitles: cache
      .filter(({ forwardLinksInBody }) =>
        forwardLinksInBody.includes(targetNoteTitle)
      )
      .map(({ title }) => title),
  }));
};

const getMetadataLines = backlinkTitles =>
  backlinkTitles.length
    ? [
        METADATA_DIVIDER,
        METADATA_TITLE,
        "### Backlinks",
        backlinkTitles
          // Must escape the slash per Bear linking mechanics
          .map(title => `\t- [[${title.replace(/\//g, "\\/")}]]`)
          .join("\n"),
        METADATA_DIVIDER,
      ]
    : null;

/**
 * If the cached note has backlinks, they are different from those in backlinkIndex,
 * the backlinks should be updated.
 */
const hasOutdatedBacklinks = ({ title, currentBacklinks }, backlinkIndex) =>
  backlinkIndex.some(
    ({ targetNoteTitle, linkSourceTitles }) =>
      targetNoteTitle === title &&
      !stringArraysHaveSameValues(linkSourceTitles, currentBacklinks)
  );

/** If the cached note has backlinks, but no entry in backlinkIndex, the backlinks are no longer valid. */
const areAllBacklinksMissing = ({ title, currentBacklinks }, backlinkIndex) =>
  currentBacklinks.length &&
  !backlinkIndex.some(({ targetNoteTitle }) => title === targetNoteTitle);

/** Returns cached note + backlink metadata for notes in cache that need to be updated. */
const getNotesChangesToPush = (cache, backlinkIndex) =>
  cache
    .filter(
      cachedNote =>
        hasOutdatedBacklinks(cachedNote, backlinkIndex) ||
        areAllBacklinksMissing(cachedNote, backlinkIndex)
    )
    .map(({ title, identifier, noteWithoutMetadata }) => {
      const backlinkIndexData = backlinkIndex.find(
        ({ targetNoteTitle }) => title === targetNoteTitle
      );
      if (!backlinkIndexData) return null;
      const { linkSourceTitles } = backlinkIndexData;
      return { identifier, noteWithoutMetadata, linkSourceTitles };
    })
    .filter(Boolean);

const createBacklinks = async () => {
  const cachedNotes = await populateCacheWithNotesWithLinks();
  await completeCacheWithNotesWithoutLinks(cachedNotes);
  const backlinkIndex = getBacklinkIndex(cachedNotes);
  const numResults = (
    await Promise.all(
      getNotesChangesToPush(cachedNotes, backlinkIndex).map(
        async ({ linkSourceTitles, identifier, noteWithoutMetadata }) =>
          await replaceBearNoteBody(
            identifier,
            [
              ...trimLines(noteWithoutMetadata.split("\n")),
              ...(getMetadataLines(linkSourceTitles) || []),
            ].join("\n")
          )
      )
    )
  ).length;
  console.log(`Backlink parsing done -- updated ${numResults} notes.`);
};

await createBacklinks();
	// Variables used by Scriptable.
	// These must be at the very top of the file. Do not edit.
	// icon-color: blue; icon-glyph: link;

	/**
	* GENERATE BEAR BACKLINKS
	*
	* This script will find and add backlinks in Bear.
	*
	* !!Please backup your notes before running!! https://bear.app/faq/Backup%20&%20Restore/
	* I haven't had any issues with running this script, but have only tested it
	* with my notes. I would strongly suggest that you back up your notes so you
	* can restore them if you don't like the outcome.
	*
	* INSTRUCTIONS
	* 1. Edit the Settings below this comment block
	* 2. Turn on "Reduce motion" setting: https://support.apple.com/en-gb/HT202655
	* - This isn't mandatory, but will speed up the execution of the script. Because
	* we have to make a roundtrip between Scriptable and Bear for each note evaluated,
	* this can take a very long time to run (I've had it run for ~30 minutes with 770 notes).
	* Turning reduce motion on significantly reduces the amount of time each roundtrip takes.
	* - [UPDATE 2020-11-11 -- the script seems to be broken in Split View, probably due to some OS or app changes]
	* -If you run this on an iPad with split view support, having Scriptble and Bear open
	* next to each other makes this run exponentially faster, as there is no app switching.-
	* 3. Run script
	* - NB! You are effectively locked out of your device while this is running. You can quit
	* the apps if you're fast enough, but it is challenging. Make sure you won't need the device
	* while this is running.
	*/

	//
	// SETTINGS
	//

	// The results of this search will be the body of notes used to find backlinks.
	// The default here shows all notes that aren't locked (which for me is all notes).
	// The search term can be tested in Bear to see which notes will be included.
	// https://bear.app/faq/Advanced%20search%20options%20in%20Bear/
	const NOTES_SEARCH_TERM = "-@locked";

	/**
	* Place token for your device between quotes below. Note that different devices have different tokens.
	* If you use this script on different devices, you can use Device.isPad(), for example, to choose the right one.
	*
	* From Bear documentation (https://bear.app/faq/X-callback-url%20Scheme%20documentation/):
	*
	* In order to extend their functionalties, some of the API calls allow an app generated token to be
	* passed along with the other parameters. Please mind a Token generated on iOS is not valid for MacOS and vice-versa.
	*
	* On MacOS, select Help → API Token → Copy Token and will be available in your pasteboard.
	*
	* On iOS go to the preferences → General, locate the API Token section and tap the cell below
	* to generate the token or copy it in your pasteboard.
	*/
	const BEAR_TOKEN = "";

	//
	//
	//
	//
	//
	//
	//
	//
	//

	//
	// HELPERS
	//

	const uniqueArray = (...arrays) => [...new Set(arrays.flatMap(i => i))];

	const noteLinkInNoteRegex = /\[\[(.+?)\]\]/g;
	/** @param {string} noteBody */
	const getNoteLinks = noteBody =>
	uniqueArray(
	noteBody
	.split("\n")
	.flatMap(line =>
	[...line.matchAll(noteLinkInNoteRegex)].map(match => match[1])
	)
	.filter(Boolean)
	);

	/** Do string arrays have same values, in any order? */
	const stringArraysHaveSameValues = (arr1, arr2) => {
	if (arr1.length !== arr2.length) return false;
	return arr1.every(arr1Val => arr2.some(arr2Val => arr1Val === arr2Val));
	};

	/**
	* Given array of strings, return array of lines removing all empty lines
	* at beginning and end of lines.
	*/
	const trimLines = lines => {
	const { firstContentLine, lastContentLine } = lines.reduce(
	(acc, line, i) => {
	const lineHasContent = Boolean(line.trim().length);
	if (acc.firstContentLine === -1 && lineHasContent)
	acc.firstContentLine = i;
	if (lineHasContent) acc.lastContentLine = i;
	return acc;
	},
	{ firstContentLine: -1, lastContentLine: -1 }
	);
	return lastContentLine === -1
	? []
	: lines.slice(firstContentLine, lastContentLine + 1);
	};

	//
	// BEAR XCALLBACK FUNCTIONS
	//

	const BASE_URL = "bear://x-callback-url";
	const getBearCallbackObject = (endpoint, params) => {
	const callbackObject = new CallbackURL(`${BASE_URL}/${endpoint}`);
	Object.entries(params).forEach(([key, val]) =>
	callbackObject.addParameter(key, val)
	);
	callbackObject.addParameter("token", BEAR_TOKEN);
	return callbackObject;
	};

	const getFullBearNote = async noteId => {
	const callback = getBearCallbackObject("open-note", {
	id: noteId,
	open_note: "no",
	});
	return await callback.open();
	};

	const getBearSearchResults = async term => {
	const callback = getBearCallbackObject("search", { term });
	const resultsRaw = await callback.open();
	return JSON.parse(resultsRaw.notes);
	};

	const replaceBearNoteBody = async (noteId, newNoteBody) => {
	const callback = getBearCallbackObject("add-text", {
	id: noteId,
	text: newNoteBody,
	mode: "replace_all",
	open_note: "no",
	});
	return await callback.open();
	};

	//
	// NOTE PARSING
	//

	const METADATA_DIVIDER = "---";
	const METADATA_TITLE = "::METADATA::";
	const METADATA_LINE_PREFIX = "\t- [[";

	/** With ability to link to sections in notes, notes w/ "/" in title must get special handling */
	const cleanNoteLink = link => {
	const hasSectionLink = /[^\\](\/.+$)/.test(link);
	const withoutSectionLink = (() => {
	if (!hasSectionLink) return link;
	const splitBySlashes = link.split("/");
	splitBySlashes.pop();
	return splitBySlashes.join("/");
	})();
	return withoutSectionLink.replace(/\\\//g, "/");
	};

	const getMetadataFromNote = note => {
	const defaultReturn = {
	noteWithoutMetadata: note,
	currentBacklinks: [],
	};
	const lines = note.split("\n");
	const metadataTitleLineIndex = lines.indexOf(METADATA_TITLE);
	if (metadataTitleLineIndex === -1) return defaultReturn;
	const closingDividerIndex = lines.findIndex(
	(line, i) => i > metadataTitleLineIndex && line === METADATA_DIVIDER
	);
	if (closingDividerIndex === -1) return defaultReturn;
	const metadataLines = lines.splice(
	metadataTitleLineIndex - 1,
	closingDividerIndex - (metadataTitleLineIndex - 1) + 1
	);
	const currentBacklinks = metadataLines
	.filter(line => line.startsWith(METADATA_LINE_PREFIX))
	.map(line => line.replace(METADATA_LINE_PREFIX, "").replace("]]", ""))
	.map(cleanNoteLink);
	return { noteWithoutMetadata: lines.join("\n"), currentBacklinks };
	};

	const getForwardLinks = (noteWithoutMetadata, noteTitle) =>
	getNoteLinks(noteWithoutMetadata)
	.map(cleanNoteLink)
	// This can happen if linking to a subsection within a note
	.filter(forwardLink => forwardLink !== noteTitle);

	/**
	* To start, get full notebody for all links that may have note links in them.
	* False positives are removed, leaving a cache of notes that link to other notes.
	* False positive note titles are logged in console; correcting this (they contain "[[")
	* can speed up the script, especially on iPhone.
	*/
	const populateCacheWithNotesWithLinks = async () => {
	const allNotesThatMayHaveNoteLinks = await getBearSearchResults("[[");
	return (
	await Promise.all(
	allNotesThatMayHaveNoteLinks.map(async ({ identifier, title }) => {
	const { note } = await getFullBearNote(identifier);
	const { noteWithoutMetadata, currentBacklinks } = getMetadataFromNote(
	note
	);
	const forwardLinksInBody = getForwardLinks(noteWithoutMetadata, title);
	const isFalsePositive = !(
	currentBacklinks.length \|\| forwardLinksInBody.length
	);
	if (isFalsePositive) {
	console.log(
	`Note "${title}" matches search "[[", but contains no note links.`
	);
	return null;
	}
	return {
	identifier,
	title,
	noteWithoutMetadata,
	forwardLinksInBody,
	currentBacklinks,
	};
	})
	)
	).filter(Boolean);
	};

	/**
	* Initial cache load only pulls from notes that contain "[[",
	* so some target notes without links in them may be missing.
	*/
	const completeCacheWithNotesWithoutLinks = async cache => {
	const allNotes = await getBearSearchResults(NOTES_SEARCH_TERM);
	const allLinkedNoteTitles = uniqueArray(
	cache.flatMap(({ currentBacklinks, forwardLinksInBody }) => [
	...currentBacklinks,
	...forwardLinksInBody,
	])
	);
	const linkedNotesNotInCache = allLinkedNoteTitles
	.filter(
	noteTitle => !cache.some(cachedNote => cachedNote.title === noteTitle)
	)
	.map(linkedNoteTitle =>
	allNotes.find(note => note.title === linkedNoteTitle)
	)
	.filter(Boolean);
	await Promise.all(
	linkedNotesNotInCache.map(async ({ identifier, title }) => {
	const { note } = await getFullBearNote(identifier);
	cache.push({
	identifier,
	title,
	noteWithoutMetadata: note,
	forwardLinksInBody: [],
	currentBacklinks: [],
	});
	})
	);
	};

	/** Re-organize cache data into pairs of link target note ID & array of source titles linking to it. */
	const getBacklinkIndex = cache => {
	const allForwardLinks = uniqueArray(
	cache.flatMap(({ forwardLinksInBody }) => forwardLinksInBody)
	);
	return allForwardLinks.map(targetNoteTitle => ({
	targetNoteTitle,
	linkSourceTitles: cache
	.filter(({ forwardLinksInBody }) =>
	forwardLinksInBody.includes(targetNoteTitle)
	)
	.map(({ title }) => title),
	}));
	};

	const getMetadataLines = backlinkTitles =>
	backlinkTitles.length
	? [
	METADATA_DIVIDER,
	METADATA_TITLE,
	"### Backlinks",
	backlinkTitles
	// Must escape the slash per Bear linking mechanics
	.map(title => `\t- [[${title.replace(/\//g, "\\/")}]]`)
	.join("\n"),
	METADATA_DIVIDER,
	]
	: null;

	/**
	* If the cached note has backlinks, they are different from those in backlinkIndex,
	* the backlinks should be updated.
	*/
	const hasOutdatedBacklinks = ({ title, currentBacklinks }, backlinkIndex) =>
	backlinkIndex.some(
	({ targetNoteTitle, linkSourceTitles }) =>
	targetNoteTitle === title &&
	!stringArraysHaveSameValues(linkSourceTitles, currentBacklinks)
	);

	/** If the cached note has backlinks, but no entry in backlinkIndex, the backlinks are no longer valid. */
	const areAllBacklinksMissing = ({ title, currentBacklinks }, backlinkIndex) =>
	currentBacklinks.length &&
	!backlinkIndex.some(({ targetNoteTitle }) => title === targetNoteTitle);

	/** Returns cached note + backlink metadata for notes in cache that need to be updated. */
	const getNotesChangesToPush = (cache, backlinkIndex) =>
	cache
	.filter(
	cachedNote =>
	hasOutdatedBacklinks(cachedNote, backlinkIndex) \|\|
	areAllBacklinksMissing(cachedNote, backlinkIndex)
	)
	.map(({ title, identifier, noteWithoutMetadata }) => {
	const backlinkIndexData = backlinkIndex.find(
	({ targetNoteTitle }) => title === targetNoteTitle
	);
	if (!backlinkIndexData) return null;
	const { linkSourceTitles } = backlinkIndexData;
	return { identifier, noteWithoutMetadata, linkSourceTitles };
	})
	.filter(Boolean);

	const createBacklinks = async () => {
	const cachedNotes = await populateCacheWithNotesWithLinks();
	await completeCacheWithNotesWithoutLinks(cachedNotes);
	const backlinkIndex = getBacklinkIndex(cachedNotes);
	const numResults = (
	await Promise.all(
	getNotesChangesToPush(cachedNotes, backlinkIndex).map(
	async ({ linkSourceTitles, identifier, noteWithoutMetadata }) =>
	await replaceBearNoteBody(
	identifier,
	[
	...trimLines(noteWithoutMetadata.split("\n")),
	...(getMetadataLines(linkSourceTitles) \|\| []),
	].join("\n")
	)
	)
	)
	).length;
	console.log(`Backlink parsing done -- updated ${numResults} notes.`);
	};

	await createBacklinks();