Calculate R and N register values for the LMT2 integer-N PLL. Skips a lot of potential solutions so will only be accurate to within a couple of kHz, but finds results very quickly, even on 8 bit embedded platforms. Some more details: http://nbviewer.ipython.org/urls/randomskk.net/u/lmt2.ipynb

lmt2.c

/*
 * lmt2.c - Calculate R and N register values for an LMT2 PLL.
 * Written in 2013 by Adam Greig <adam@adamgreig.com>
 *
 * To the extent possible under law, the author has dedicated all copyright
 * and related and neighboring rights to this software to the public domain
 * worldwide. This software is distributed without any warranty.
 *
 * For details, see http://creativecommons.org/publicdomain/zero/1.0/
 *
 */

/* Explicit integer sizes since we are targetting embedded platforms. */
#include <stdint.h>

/* Just for printf() and atoi() in main(), not needed for find_r_n(). */
#include <stdio.h>
#include <stdlib.h>

/*
 * Find the integers R and N (1<=R<=1300, 64<=N<=65535) so that F=13MHz * N/R
 * is close to the target frequency freq_target_khz (specified in kHz).
 * Should find perfect solutions if they exist. Otherwise might not find the
 * best possible solution, but will likely be within a couple kHz and computes
 * quickly and with little memory usage.
 *
 * Pass the target frequency as an unsigned 32 bit integer, and pass pointers
 * to unsigned 16 bit integers to be filled with the best values of R and N.
 * See main() below for a usage example.
 *
 * How? Most generally there are very many combinations of R and N (around 85
 * million) which we do not want to search exhaustively. Since the relationship
 * is linear, all real solutions lie on the straight line N=(F*R)/(13MHz).
 * Good integer solutions lie close to this line, so we'll just consider
 * truncated values on the line. Further, perfect integer solutions will
 * necessarily require that R is a multiple of 13 (since 13 is prime and
 * appears in the denominator of the linear equation). This means there are
 * only 100 possible candidates for R that are within bounds, each of which has
 * a corresponding N through the linear equation, so we can check each one and
 * select the pair that gives the lowest error.
 */
void find_r_n(uint32_t freq_target_khz, uint16_t *best_r, uint16_t *best_n)
{
    uint32_t freq_result_khz;
    uint16_t n;
    int16_t error, best_error = 0x7FFF;
    uint8_t r;

    for(r=(uint8_t)1; r<=(uint8_t)100; r++) {
        n = (freq_target_khz * r + (uint16_t)500) / (uint16_t)1000;
        freq_result_khz = (((uint32_t)1000 * n) / r);
        error = freq_result_khz - freq_target_khz;
        if(error < (int16_t)0) error = -error;
        if(error < best_error) {
            *best_n = n;
            *best_r = r * (uint16_t)13;
            best_error = error;
            if(error == (int16_t)0) return;
        }
    }
}

/* Usage example for find_r_n. Takes a target frequency (in Hz) from the
 * command line as the first and only argument, then prints the resulting R and
 * N values, along with the corresponding frequency error.
 */
int main(int argc, char* argv[])
{
    uint32_t freq_target_hz, freq_target_khz, freq_out, error_out;
    uint16_t best_r = 0, best_n = 0;

    if(argc != 2) {
        printf("Usage: %s <frequency to find N and R for, in Hz>\n", argv[0]);
        return 1;
    }

    freq_target_hz = atoi(argv[1]);
    freq_target_khz = freq_target_hz / 1000;
    printf("Target: %ukHz\n", freq_target_khz);

    find_r_n(freq_target_khz, &best_r, &best_n);

    freq_out = (13000000L * (uint32_t)best_n) / (uint32_t)best_r;
    error_out = abs(freq_out - (freq_target_hz));
    printf("Optimum solution: n=%u, r=%u, giving f=%uHz (error=%uHz)\n",
           best_n, best_r, freq_out, error_out);

    return 0;
}