Skip to content

Instantly share code, notes, and snippets.

@wdalmut
Created January 6, 2012 16:53
Show Gist options
  • Save wdalmut/1571403 to your computer and use it in GitHub Desktop.
Save wdalmut/1571403 to your computer and use it in GitHub Desktop.
couchdb bulk operation using geonames database
<?php
$filename = dirname(__FILE__) . '/allCountries.txt';
$file = new SplFileObject($filename ,'r');
$map = array(
0 => 'id',
1 => 'name',
2 => 'asciiName',
3 => 'alternateNames',
4 => 'latitude',
5 => 'longitude',
6 => 'featureClass',
7 => 'featureCode',
8 => 'countryCode',
9 => 'cc2',
10 => 'admin1Code',
11 => 'admin2Code',
12 => 'admin3Code',
13 => 'admin4Code',
14 => 'population',
15 => 'elevation',
16 => 'gTopO30',
17 => 'timezone',
18 => 'modificationDate'
);
$timeStart = microtime(true);
$ch = curl_init('http://couchdb.local:5984/geonames_bulk/_bulk_docs');
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_CUSTOMREQUEST, "POST");
curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/json'));
$index = 0;
$data = array();
while (!$file->eof()) {
$document = array_combine($map, $file->fgetcsv("\t"));
$data["docs"][] = $document;
if ($index % 100 == 0) {
$data = json_encode($data);
curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
$returned = curl_exec($ch);
if (!$returned) {
exit("NOT RUN");
} else {
echo ".";
}
$data = array();
}
++$index;
}
$timeEnd = microtime(true);
$timeDiff = $timeEnd - $timeStart;
echo "time taken : {$timeDiff} s";
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment