Skip to content

Instantly share code, notes, and snippets.

@senz
Last active June 11, 2016 18:57
Show Gist options
  • Save senz/55c80d29a6ac256387e8890d6f469b16 to your computer and use it in GitHub Desktop.
Save senz/55c80d29a6ac256387e8890d6f469b16 to your computer and use it in GitHub Desktop.
P2P i-Blocklist generator uses unix gzip and sort
<?php
return [
'list' => [
['http://list.iblocklist.com/?list=ydxerpxkpcfqjaybcssw&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'bluetack-level1'],
['http://list.iblocklist.com/?list=gyisgnzbhppbvsphucsw&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'bluetack-level2'],
['http://list.iblocklist.com/?list=plkehquoahljmyxjixpu&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'bluetack-suspicious'],
['http://list.iblocklist.com/?list=xshktygkujudfnjfioro&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'bluetack-microsoft'],
['http://list.iblocklist.com/?list=gihxqmhyunbxhbmgqrla&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'blutack-bogon'],
['http://list.iblocklist.com/?list=lujdnbasfaaixitgmxpp&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'cidr-report.org-bogon'],
['http://list.iblocklist.com/?list=cwworuawihqvocglcoss&fileformat=p2p&archiveformat=gz', 'p2p', 'gz', 'bluetack-badpeers'],
],
'sortOut' => '/srv/www/blocklist/public/outsorted'
];
<?php
/**
* @author Konstantin G Romanov
*/
if (!defined('STDERR') || !defined('STDOUT')) {
echo "Must be ran in console\n";
exit(1);
}
$configPath = getenv('BLOC_CONFIG');
if (!$configPath || !is_readable($configPath)) {
echo "BLOC_CONFIG is not set or readable";
exit(1);
}
$config = require $configPath;
function logstr($msg, $error = false, $insertnl = true) {
fwrite($error ? STDERR : STDOUT, $msg . ($insertnl ? "\n" : ''));
}
function sortTextFile($name) {
$seeker = new TextLineSeeker($name);
$total = $seeker->getTotalLines();
$pivot = floor($total / 2);
}
$curl = curl_init();
$linecount = 0;
$bufflength = 1024;
$bigoutname = tempnam(sys_get_temp_dir(), 'out');
$bigout = fopen($bigoutname, 'w+');
logstr("output name: " . $bigoutname);
$downmap = [];
foreach ($config['list'] as $item) {
list($url, $format, $compression, $desc) = $item;
$outfilename = tempnam(sys_get_temp_dir(), 'block-');
$outfile = fopen($outfilename, 'w');
$downmap[$url] = [$outfile, $outfilename];
curl_setopt_array($curl, [
CURLOPT_FILE => $outfile,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_URL => $url,
CURLOPT_TIMEOUT => 120,
]);
logstr("Downloading " . $desc . "...", false, false);
$result = curl_exec($curl);
logstr("done.");
$info = curl_getinfo($curl);
if (!$result) {
logstr("exec error: " . curl_error($curl), true);
logstr(print_r($info, true), true);
unlink($outfilename);
unset($downmap[$url]);
logstr("skipping.");
continue;
}
if ('gz' == $compression) {
logstr("decompressing");
$gzh = gzopen($outfilename, 'r');
while (!feof($gzh)) {
$buff = gzgets($gzh);
if ($buff[0] != '#' && !empty(trim($buff)))
fwrite($bigout, $buff);
logstr(".", false, false);
$linecount++;
}
echo "\n";
fclose($outfile);
}
}
C_ALL='C' sort -u $bigoutname", 'r');
while ( !feof($sorthandle) ) {
$buff = fread($sorthandle, 1024);
$res = fwrite($sortedouthandle, $buff);
if (!$res) throw new Exception("Write failure");
logstr('.', false, false);
}
logstr('done');
fclose($sortedouthandle);
unlink($bigoutname);
system('gzip -f ' . $config['sortOut']);
logstr("Total lines: " . $linecount);
logstr("DONE");
interface LineSeekable {
/**
* @return int
*/
function getCurrentLineNo();
/**
* @param int $line
* @return void
*/
function setLineNo(int $line);
/**
* @return int
*/
function getTotalLines();
/**
* @return string
*/
function getLine();
/**
* @param int $line1
* @param int $line2
* @return void
*/
function exchangeLines( int $line1, int $line2 );
/**
* @param int $from
* @param int $to
* @return void
*/
function moveLine (int $from, int $to );
}
class TextLineSeeker implements LineSeekable {
private $fileHandle;
private $currentLine;
private $totalLines;
/**
* TextLineSeeker constructor.
* @param $fileName
*/
public function __construct( $fileName ) {
if (!is_readable($fileName)) throw new InvalidArgumentException("fileName must be a readable file");
$this->fileHandle = fopen($fileName, 'r+');
rewind($this->fileHandle);
$this->currentLine = 0;
$this->totalLines = 0;
while (!feof($this->fileHandle) && fgets($this->fileHandle) !== false) $this->totalLines++;
}
function getCurrentLineNo() {
return $this->currentLine;
}
function setLineNo( int $line ) {
rewind($this->fileHandle);
$cline = 0;
while (!feof($this->fileHandle) && $cline < $line) {
fgets( $this->fileHandle );
$cline++;
}
if ($line != $cline) throw new Exception("Line not found");
$this->currentLine = $line;
}
function getTotalLines() {
return $this->totalLines;
}
function getLine() {
$line = fgets($this->fileHandle);
$res = fseek($this->fileHandle, -(strlen($line) + 1), SEEK_CUR);
if (0 !== $res) throw new Exception("Failed to set file cursor");
return $line;
}
function exchangeLines( int $line1, int $line2 ) {
$this->setLineNo($line1);
$str1 = $this->getLine();
$this->setLineNo($line2);
$str2 = $this->getLine();
fwrite($this->fileHandle, $str1);
$this->setLineNo($line1);
fwrite($this->fileHandle, $str2);
}
function moveLine( int $from, int $to ) {
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment