public
Last active

Taproot post-storage code as of 2013-12-08. Working live on waterpigs.co.uk, probably a little too rusty/specific for convenient use elsewhere but maybe of interest. storage.php is basic functional library — currently it depends on symfony/yaml for post serialisation but that can be replaced with any function capable of serializing an array structure, e.g. json_encode/decode. FifoQueue.php is a small FIFO queue implementation, used when querying the index Storage.php is the Taproot\Storage class, an alternative to creating partially-applied closures from the basic functions, with some sane defaults built in. example.php is a small demo of the API for storing/indexing/fetching/querying items. Currently the query method returns an array, my next job is to make a small class which lazily loads posts as they’re actually requested, which should make some interesting things possible, e.g. querying over multiple instances of Storage at the same time.

  • Download Gist
FifoQueue.php
PHP
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42
<?php
 
namespace Taproot;
 
use SplQueue;
 
class FifoQueue extends SplQueue {
protected $capacity;
public function __construct($capacity) { $this->capacity = $capacity; }
function enqueue($item) {
$this->push($item);
if ($this->count() > $this->capacity)
$this->dequeue();
}
function each($callback) {
$out = [];
foreach ($this as $item)
$out[] = $callback($item);
return $out;
}
function atCapacity() {
return $this->count() == $this->capacity;
}
function second() {
return $this->offsetExists(1) ? $this->offsetGet(1) : null;
}
function penultimate() {
$o = $this->capacity - 2;
return $this->offsetExists($o) ? $this->offsetGet($o) : null;
}
function last() {
$o = $this->capacity - 1;
return $this->offsetExists($o) ? $this->offsetGet($o) : null;
}
}
Storage.php
PHP
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63
<?php
 
namespace Taproot;
 
use Taproot\Storage;
 
class Storage {
protected $basepath;
protected $toIndexRow;
protected $columns;
protected $indexSorter;
protected $filters;
protected $idKey;
public function __construct($basepath, $toIndexRow, $columns=['id', 'published'], $indexSorter=null, $filters=null, $idKey='id') {
$this->basepath = $basepath;
$this->toIndexRow = $toIndexRow;
$this->columns = $columns;
$this->idKey = $idKey;
if (!file_exists($basepath))
@mkdir($basepath);
if ($indexSorter === null)
$indexSorter = function ($a, $b) {
return ($a[1] - $b[1]) * -1;
};
$this->indexSorter = $indexSorter;
if ($filters === null) {
$filters = [];
if (($key = array_search('tagged', $columns)) !== false)
$filters[] = Storage\makeTaggedFilter($key);
if (($key = array_search('deleted', $columns)) !== false)
$filters[] = Storage\makeDeletedFilter($key);
}
$this->filters = $filters;
}
public function get($id) {
return Storage\get($this->basepath, $id);
}
public function put($item) {
return Storage\put($this->basepath, $item, $this->idKey);
}
public function index($item) {
return Storage\index($this->basepath, $this->columns, $this->toIndexRow, $this->indexSorter, $item);
}
public function reindex() {
return Storage\reindex($this->basepath, $this->columns, $this->toIndexRow, $this->indexSorter);
}
public function query(array $query) {
return Storage\query($this->basepath, $query, $this->filters);
}
public function prevNextItems($idOrItem) {
return Storage\prevNextItems($this->basepath, $idOrItem);
}
}
example.php
PHP
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
<?php
 
use Taproot\Storage;
 
function toIndexRow($note) {
$timestamp = Storage\toDateTime($note['published'])->format('U');
return [$note['id'], $timestamp, ' ' . implode(' ', $note['tags']) . ' ', !empty($note['deleted'])];
}
 
$notes = new Storage($basepath, 'toIndexRow', ['id', 'published', 'tagged', 'deleted']);
 
// Now arbitrary array structures can be saved and queried
$note = [
'id' => '1',
'content' => 'Hello world!',
'published' => '2013-12-08 12:00:00',
'tags' => ['test', 'dev']
];
$notes->put($note);
$notes->index($note);
 
$note = $notes->get('1');
 
// by default posts can be queried by tags ('tagged' => 'tag') and deleted status
// limit sets maximin number of results, before/after allow for crude-but-stable datetime-based pagination
// query function/method returns array with results, then newer/older notes if they exist, or null if they don’t exist
list($notes, $olderNote, $newerNote) = $notes->query(['limit' => 20]);
storage.php
PHP
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
<?php
 
namespace Taproot\Storage;
 
use DateTime;
use Symfony\Component\Yaml\Yaml;
use Taproot\FifoQueue;
 
function toDateTime($dt) {
return $dt instanceof Datetime ? $dt : new DateTime($dt);
}
 
function get($basepath, $id) {
$notepath = $basepath . DIRECTORY_SEPARATOR . $id . '.yaml';
if (file_exists($notepath) and $id !== null)
return Yaml::parse(file_get_contents($notepath));
else
return null;
}
 
// given a post, saves it
function put($basepath, array $note, $idKey='id') {
$notepath = $basepath . DIRECTORY_SEPARATOR . $note[$idKey] . '.yaml';
return file_put_contents($notepath, Yaml::dump($note));
}
 
// given a post, updates the relevant indexes
function index($basepath, $columns, $toIndexRow, $sorter, array $note) {
$indexFile = fopen($basepath . DIRECTORY_SEPARATOR . 'index.csv', 'c+');
$csvColumns = fgetcsv($indexFile);
$rowToSave = $toIndexRow($note);
$index = [];
while (($row = fgetcsv($indexFile)) !== false) {
if ($rowToSave[0] == $row[0]) {
$index[] = $rowToSave;
$updated = true;
} else {
$index[] = $row;
}
}
fclose($indexFile);
if (!isset($updated))
array_unshift($index, $rowToSave);
usort($index, $sorter);
$indexFile = fopen($basepath . DIRECTORY_SEPARATOR . 'index.csv', 'w');
fputcsv($indexFile, $csvColumns ?: $columns);
foreach ($index as $row)
fputcsv($indexFile, $row);
fclose($indexFile);
return true;
}
 
// reindexes all items in this basepath
function reindex($basepath, $columns, $toIndexRow, $sorter) {
$index = [];
 
$noteIds = array_map(function ($p) { return pathinfo($p, PATHINFO_FILENAME); }, glob($basepath . '/*.yaml'));
$notes = array_map(function ($id) use ($basepath) { return get($basepath, $id); }, $noteIds);
 
foreach ($notes as $note) {
$index[] = $toIndexRow($note);
}
 
usort($index, $sorter);
 
$indexFile = fopen($basepath . DIRECTORY_SEPARATOR . 'index.csv', 'w');
fputcsv($indexFile, $columns);
foreach ($index as $row)
fputcsv($indexFile, $row);
fclose($indexFile);
}
 
// a comparator for ordering by published datetime
function publishedComparator($dir) {
$dir = strpos($dir, 'asc') == 0 ? 1 : -1;
return function($a, $b) use ($dir) {
$ats = toDateTime($a['published'])->format('U');
$bts = toDateTime($b['published'])->format('U');
return ($bts - $ats) * $dir;
};
}
 
// returns a filter function allowing rows to be queried on the presence of a string within a space-separated string
function makeTaggedFilter($rowIndex = 2, $queryKey = 'tagged') {
return function ($row, $query) use ($rowIndex, $queryKey) {
if (!empty($query[$queryKey]))
return strpos(" {$row[$rowIndex]} ", " {$query[$queryKey]} ") !== false;
return true;
};
}
 
// syntatic sugar for common case of boolean filter
function makeDeletedFilter($rowIndex = 3) { return makeBooleanFilter($rowIndex, true); }
 
/**
* In : 1 0
* Inv: 1 0 1
* : 0 1 0
*/
function makeBooleanFilter($rowIndex, $invert = false) {
return function ($row, $query) use ($rowIndex, $invert) {
return $invert == empty($row[$rowIndex]);
};
}
 
// returns a filter function allowing rows to be queried on equality to a given value
function makeEqualityFilter($rowIndex, $queryKey) {
return function ($row, $query) use ($rowIndex, $queryKey) {
if (!empty($query[$queryKey]))
return $query[$queryKey] == $row[$rowIndex];
return true;
};
}
 
// queries items based on a query and a list of filters
// pagination behaviour is built-in and hardcoded to certain indexes, this is bad and should be made more flsxible
function query($basepath, array $query, array $filters = []) {
$itemFilter = count($filters) === 0
? function () { return true; }
: function ($row) use ($query, $filters) {
foreach ($filters as $filter) {
if ($filter($row, $query) === false) return false;
}
return true;
};
 
if (!isset($query['limit']))
$query['limit'] = 20;
$itemQueue = new FifoQueue($query['limit'] + 2);
$itemQueue->enqueue(null); // pre-fill the newer item buffer
$indexFile = fopen($basepath . "/index.csv", 'c+');
fgetcsv($indexFile);
if (isset($query['after'])) {
$after = Text\toDateTime($query['after'])->format('U');
$break = function ($itemQueue) use ($after) {
if ($itemQueue->atCapacity() and $itemQueue->last() !== null and $itemQueue->last()[1] <= $after) {
return true;
}
};
} else {
$before = isset($query['before']) ? Text\toDateTime($query['before']) : new DateTime;
$before = $before->format('U');
$break = function ($itemQueue) use ($before) {
if ($itemQueue->atCapacity() and $itemQueue->second() !== null and $itemQueue->second()[1] < $before) {
return true;
}
};
}
while (($row = fgetcsv($indexFile)) !== false) {
if (!$itemFilter($row)) continue;
$itemQueue->enqueue($row);
if ($break($itemQueue)) break;
}
fclose($indexFile);
$items = iterator_to_array($itemQueue);
if (count($items) > $query['limit']) {
$olderBuffer = array_pop($items);
$newerBuffer = array_shift($items);
$olderItem = $olderBuffer !== null ? get($basepath, $olderBuffer[0]) : null;
$newerItem = $newerBuffer !== null ? get($basepath, $newerBuffer[0]) : null;
} else {
$olderItem = null;
$newerItem = null;
}
$items = array_filter($items);
$items = array_map(function ($r) use ($basepath) { return get($basepath, $r[0]); }, $items);
return [$items, $olderItem, $newerItem];
}
 
// given an item id, return an array of [prevItem, nextItem] where each might be null if it doesn’t exist
// warning: calling this with a nonexistant id produces undefined behaviour
function prevNextItems($basepath, $itemOrId) {
$id = is_string($itemOrId) ? $itemOrId : $itemOrId['id'];
$indexFile = fopen($basepath . '/index.csv', 'r');
$noteQueue = new FifoQueue(3);
// to do: make this happen by default on queue creation
$noteQueue->enqueue(null); $noteQueue->enqueue(null); $noteQueue->enqueue(null);
fgetcsv($indexFile);
while (($row = fgetcsv($indexFile)) !== false) {
$noteQueue->enqueue($row);
if ($noteQueue->penultimate()[0] == $id)
break;
}
fclose($indexFile);
return [get($basepath, $noteQueue->last()[0]), get($basepath, $noteQueue[0][0])];
}
 
// Taproot-specific syntatic sugar for generating pagination links from query results
function paginationLinks($items, $olderItem, $newerItem, $request, $app, $url) {
if ($newerItem !== null) {
$q = $request->query->all();
unset($q['before']);
$q['after'] = Text\toDateTime($items[0]['published'])->format(DateTime::W3C);
$newerLink = $app['url_generator']->generate($url, $q);
} else {
$newerLink = null;
}
 
if ($olderItem !== null) {
$q = $request->query->all();
unset($q['after']);
$q['before'] = Text\toDateTime($items[count($items) - 1]['published'])->format(DateTime::W3C);
$olderLink = $app['url_generator']->generate($url, $q);
} else {
$olderLink = null;
}
return [$olderLink, $newerLink];
}

Please sign in to comment on this gist.

Something went wrong with that request. Please try again.