Skip to content

Instantly share code, notes, and snippets.

@konark-cliqz

konark-cliqz/human-web.es Secret

Last active May 9, 2021 09:32
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save konark-cliqz/fd0e488f1ac691bbd5f557b59fb65766 to your computer and use it in GitHub Desktop.
Save konark-cliqz/fd0e488f1ac691bbd5f557b59fb65766 to your computer and use it in GitHub Desktop.
Download ZIP
human-web code base
Raw
human-web.es
import CliqzEvents from "core/events";
import CliqzBloomFilter from "human-web/bloom-filter";
import { utils } from "core/cliqz";
import md5 from "core/helpers/md5";
import ResourceLoader from 'core/resource-loader';
import { queryActiveTabs } from 'core/tabs';
import { forEachWindow } from 'platform/browser';
import CliqzSecureMessage from 'hpn/main';
Components.utils.import("resource://gre/modules/Services.jsm");
Components.utils.import("resource://gre/modules/FileUtils.jsm");
Cu.import('resource://gre/modules/XPCOMUtils.jsm');
const dnsService = Components.classes["@mozilla.org/network/dns-service;1"]
.createInstance(Components.interfaces.nsIDNSService);
var nsIAO = Components.interfaces.nsIHttpActivityObserver;
var nsIHttpChannel = Components.interfaces.nsIHttpChannel;
var refineFuncMappings ;
var gadurl = /\.google..*?\/(aclk)\?/;
/*
Configuration for Bloomfilter
*/
var bloomFilterSize = 500001;
var falsePositive = 0.01;
var bloomFilterNHashes = 7;
function _log(msg){
try{
if(CliqzHumanWeb.debug) {
CliqzUtils.log(msg, CliqzHumanWeb.LOG_KEY);
}
}
catch(e){CliqzUtils.log(e, CliqzHumanWeb.LOG_KEY)};
}
function getRandomIntInclusive(min, max) {
min = Math.ceil(min);
max = Math.floor(max);
return Math.floor(Math.random() * (max - min + 1)) + min;
}
function getHTML(...args) {
return utils.callAction('core', 'getHTML', args);
}
var CliqzHumanWeb = {
VERSION: '2.4',
WAIT_TIME: 2000,
LOG_KEY: 'humanweb',
debug: false,
httpCache: {},
httpCache401: {},
queryCache: {},
privateCache: {},
UrlsCache :false,
docCache: {},
strictMode: false,
qs_len:30,
rel_part_len:18,
rel_segment_len:15,
doubleFetchTimeInSec: 3600,
can_urls: {},
deadFiveMts: 5,
deadTwentyMts: 20,
msgType:'humanweb',
probHashLogM: [[-1.839225984234144, -1.8009413231413045, -2.5864601561900273, -2.779077348461893, -2.41154163187108, -2.8701669802592216, -3.010853183897427, -2.9831997811803244, -3.0258844950086354, -3.028549665472028, -4.922879838696041, -4.503195676571833, -4.598154161855354, -5.027344698225922, -5.217723368153992, -5.366375752342653, -5.195909278900248, -3.2940850310881387, -5.181468249477294, -6.110371416838184, -5.766202229535159, -5.393352786265537, -4.920024681004049, -6.049294408676102, -6.399015490326156, -4.690163484884073, -7.459851804835359, -5.825337077308845, -4.675000760327133, -5.3635239851672845, -6.331713706782754, -6.169284763337202, -5.951151191317628, -5.768078242705815, -7.332129653597336, -6.766704624275413, -6.01222435569866, -5.268617743244878, -6.279416117827666, -5.861972559492891, -5.838071468059843, -6.267250173459698, -6.283981033030117, -6.454759516784266, -6.5195694451700765, -6.83526729571271, -6.458157413359671, -6.359696067797872, -6.250241538141395, -6.647172527007435, -6.973987597161365, -6.4598607011160425, -7.685547265665763, -6.5912026847359835, -5.794718807999361, -6.474629988987015, -7.135098972554303, -6.880960523371609, -6.747429130747086, -7.733118660641065, -8.398636456057481, -7.6323973914922005], [-2.11332750230783, -2.3060786022344897, -2.3740965556720726, -2.5398307619465608, -2.463417892707307, -2.130953109899374, -2.8009573227909073, -2.9401508827086436, -2.748847204575824, -2.8414237914951976, -5.010884149435608, -4.7716590964777925, -4.868359687249472, -5.322380370933768, -5.037006822424893, -5.279762905036231, -5.635270908532894, -2.9219817650555973, -5.50179037155463, -6.5406369920023, -6.311742070591071, -5.516975693542294, -5.7251712834206305, -6.083331008626942, -6.409455776722731, -4.856277028595425, -7.356357334609669, -6.255977591877104, -5.042089425488359, -5.723688703460408, -6.9306288690141455, -5.879866127104103, -6.269949621534028, -6.969512838475713, -7.475373123729266, -6.802062729887527, -6.566109586768108, -6.661695002244713, -6.634070998648396, -6.128265317507457, -6.0654489711795785, -6.79728322915323, -6.99559673771516, -6.996654379273295, -7.671972036661277, -7.840255377779173, -7.132455920432357, -7.233113706563278, -6.737551430374816, -7.2706736012479585, -7.430290364348156, -6.991903776514698, -8.559572276072503, -7.231105001502197, -6.117005677632925, -6.669675284003577, -7.87022016648111, -7.635233211015519, -7.18343028641663, -8.53463332772525, -8.567176875457722, -7.573077285525099], [-1.6313545940843748, -2.457209014039589, -2.565380346827546, -2.767788786784649, -2.6386222432933284, -2.595661570703591, -2.6893427324813977, -2.7198733792574403, -2.720533202987637, -2.8467380805237736, -5.029918409868431, -4.178710463028381, -5.07588590483658, -5.057436988529895, -4.899938336310833, -5.271744381251988, -5.715419865537325, -3.1739071041721116, -5.367102243680773, -6.224018730172038, -5.866973141306898, -5.730015933050199, -5.360958031920617, -6.20201967617667, -6.328452890969734, -4.837569916970598, -7.520833339090353, -5.8325529174204185, -4.950768880962046, -5.339171801246006, -6.500538943869661, -6.141626179853517, -5.826929659866057, -6.371549418153106, -7.095602530626063, -6.863928261303842, -6.26532998038965, -6.486382994639529, -6.568804701240378, -5.928030195562504, -6.138501621140467, -6.375855129864318, -6.672567424416514, -6.87690014131182, -7.500698430681298, -7.486404619424152, -7.110483757622713, -6.948864783187929, -6.7689909475117735, -7.44680608044408, -7.305175362028806, -6.490004000406482, -8.278142463867583, -7.100043692626031, -6.262757631704339, -6.785644569035485, -7.917859578552653, -6.966883288690608, -6.7663306639529575, -7.827911341889713, -7.709275282211631, -7.650980633730635], [-2.2483023852308777, -2.2601431921520803, -2.543211818297523, -2.6290975996017654, -2.597012293697696, -2.425038774903284, -2.6464962853598264, -2.7268820909874645, -2.7170988469992086, -2.6384412943652213, -4.8059842634060495, -4.801920790322523, -4.881995682865955, -4.399130557719554, -4.977525552072196, -5.254333597613871, -5.4861286120783905, -3.172333855510997, -5.311009967578493, -6.3787775808955045, -6.060833750357407, -5.179488625881995, -5.284849141539821, -6.185226477248653, -6.342526765978808, -5.0477193482508715, -7.229508117457252, -5.92913163950562, -4.781563610047119, -5.175693021409531, -6.686598785853124, -5.953265540842049, -5.765108625614716, -6.249636653394711, -7.053405713688212, -6.871084156894258, -4.746700678954438, -4.435206614193364, -6.2492260600816145, -5.437566316202071, -5.90294982336378, -6.5708663077474965, -6.4774110336687505, -6.642451113295277, -7.490560332753359, -7.638809897851904, -7.037026821604173, -6.792453878834064, -6.557926605402683, -7.183535297458448, -7.556615154408894, -6.675849393697125, -7.929241490870054, -6.920447880012594, -5.937211886970303, -6.536908132533395, -7.266281062975223, -7.138583909596181, -6.9407325511841, -7.611188320541975, -8.063402408224212, -7.7539194100649205], [-2.0350349956205984, -2.348602646771946, -2.41594411405956, -2.651370395735765, -2.563410442434685, -2.706962611141542, -2.7917112923152776, -2.7483543933766477, -2.6785026736567357, -2.8111298756441037, -4.759837736546805, -4.20618015484438, -4.787453629928326, -4.659538126267765, -5.015532806050122, -5.056335259588051, -5.339468177159593, -3.040667184111712, -5.203144447478613, -6.2264472230490675, -5.7097357671058155, -5.585194192596145, -5.035187712987706, -5.542988699592872, -6.206189420830567, -4.864552817623237, -7.387161020524848, -5.130725109803191, -4.710070938057912, -5.015532806050122, -5.563665939332496, -5.8412571088023135, -5.744609136156329, -6.206575148897066, -6.842735398083401, -6.794361813799185, -5.850672107736916, -6.088449236501511, -6.348519254409916, -5.725588459198469, -6.064735388153122, -6.479956671206651, -6.2209527037314265, -6.70094736915976, -7.3405268040104845, -7.140183173146506, -6.467358872953416, -6.7233339772218645, -6.184068000080206, -6.381516598393398, -7.406185056529545, -6.368731036096427, -7.402351186018824, -6.6091509166536255, -5.9623588984476195, -6.352529061520146, -7.133337180638517, -7.24783614109371, -6.901266731308386, -7.234806640803376, -7.432126848507545, -7.494477663675224], [-2.0215812860615885, -2.455637406643175, -2.4330253990141046, -2.6113902739466517, -2.6033787533338284, -2.6337646574813993, -2.7111759743895725, -2.7471499108067454, -2.721672070749707, -2.7644985821403028, -4.771735346669196, -4.967298931497775, -4.806933772233218, -4.796223062219612, -4.702049659123834, -4.615986584200591, -5.422222957790021, -3.024083897225106, -5.341473161524181, -5.691932137893965, -5.870247680316981, -5.6516120254419615, -4.85258639108815, -5.9684706438872475, -6.205102036008844, -4.762283950640148, -7.106211026775939, -5.754565801069223, -4.28552037163959, -5.083154204506742, -6.514533306380853, -5.91282033178054, -5.801594696670316, -6.211408552601299, -6.884100000075919, -6.5060032575028, -6.051550266183317, -6.1528452495165125, -6.237040028392324, -5.845664774991169, -5.83750622157883, -6.376088248894053, -6.3936457603588925, -6.58430154569624, -7.071724850704769, -7.213982427018076, -6.56730052905341, -6.608473779316545, -6.403307671270629, -6.973970456584425, -7.485700648480843, -6.5060032575028, -7.126013654072119, -6.592294210440276, -5.525669597265563, -6.0726189713973024, -6.168884085714272, -7.057863242618145, -6.702467385782517, -7.84259108644045, -7.932742183434748, -6.889071012797939], [-2.3515175517946965, -2.4895377785815813, -2.455794452354539, -2.6177065611128625, -2.3916765698655924, -2.55893036105207, -2.621684157393488, -2.6988517931658524, -2.630805311213068, -2.6634082117951667, -4.639123440452939, -4.108116635400416, -4.850674635561831, -4.879642999369406, -5.020506071184806, -5.079702479761731, -5.3856932140647995, -2.922465393995544, -5.272571266829349, -6.095451326503016, -5.703507123198923, -5.5861660464475555, -5.144197286007818, -5.722031888813277, -6.28586346876822, -4.872555039237073, -6.886991750607913, -5.521389523166842, -4.712785087276782, -4.712557943607434, -6.258849400356963, -5.781080159458192, -5.683518571573409, -6.497978247391754, -6.801813554699944, -6.640350228026399, -6.23670827447975, -6.480528140170159, -6.503409353749256, -6.035279500709713, -6.187868456644626, -6.634900623258835, -6.227883924459252, -6.665648019797556, -7.5457712876636505, -7.422426740346121, -6.235666064901307, -6.769319721223498, -6.343344078699495, -6.975730165249003, -7.353206363454935, -6.635677322327007, -7.937813375439674, -6.762240045635436, -6.147006740966892, -6.741296871790193, -7.715813334827543, -7.183705495090042, -6.938232467377743, -7.593211012735211, -8.021194984378726, -7.326496213127361], [-2.404503840183417, -2.43761707422273, -2.4108252303609095, -2.2999425986283435, -2.542843475674806, -2.6029429346724786, -2.584141922741018, -2.5855127818195496, -2.5510561981552526, -2.647681869737238, -4.782820501953572, -4.808208666935694, -4.979563332219715, -5.008211525808261, -5.046432738628459, -5.142457459370653, -5.485491564024772, -3.1144173778919617, -5.417825855041213, -6.245547233803318, -5.938268453446564, -5.963941744764968, -5.450843208258146, -6.059373833665894, -6.223182084343739, -4.942745700375246, -7.208409268623897, -6.038143491827719, -4.6277437642171355, -4.892513996642899, -6.694766741586792, -6.117821568685912, -5.949820065666706, -6.502648045547855, -6.835922059321975, -6.694766741586792, -6.361312919572388, -6.403294392961231, -6.118769885494771, -5.676056523505988, -6.406451957946552, -6.5244722133881785, -6.6333977952105005, -6.6574953467895615, -7.573786078663716, -7.446115322000925, -6.883692335590661, -6.903250344490318, -6.3498910941692, -6.905331510694142, -7.588115904218541, -6.879623153983935, -8.035692497676832, -6.939233062369824, -5.959077527634608, -6.324513515108117, -7.549688527084656, -7.120692921566297, -6.988939594552065, -7.891731808755324, -7.727259481120012, -7.430143376434873], [-2.2924692182301323, -2.450539739657926, -2.3769703926915984, -2.532604294663703, -2.55393918066343, -2.609718771364289, -2.6246425013179655, -2.661744915258029, -2.6011009513028984, -2.6236780353341946, -4.669477044233787, -4.1521285006409165, -4.955491059584871, -4.818890308190988, -5.030853379357302, -5.002039294312803, -5.278250342600015, -3.076829272811893, -5.307731406455737, -5.826171280206974, -6.022604681193609, -5.8364861502494865, -5.222160875948972, -5.86734719439116, -6.263453577434854, -5.1217643044016885, -7.32073988901422, -5.758783848671988, -4.666476541899311, -5.259001445170042, -6.374002568031498, -6.083417735921755, -5.695962812152638, -6.490715185357981, -6.417147740477192, -6.55681384079998, -5.431378631989581, -5.843302320137157, -6.369727752361356, -5.638321273173884, -6.50456582229188, -6.275027142828877, -6.210627212225794, -6.774279201200845, -7.54625217416398, -7.377527245335997, -6.968174323388823, -6.961544464850153, -6.5738576027632725, -6.808653231033201, -7.247759185607984, -6.667823395181603, -8.06567857870257, -6.862038055489511, -5.923075085846576, -6.772454383176216, -7.732534132174032, -6.980444415980637, -6.935456028765855, -7.926350068393722, -7.409760739910981, -7.357691884279851], [-2.395140177772492, -2.3972905877762423, -2.4380054501613637, -2.59514809923619, -2.5304957080424493, -2.553494441315067, -2.592658670737186, -2.6234135177213647, -2.5068257242437872, -2.493912396989218, -4.415128005729072, -4.8162395242315545, -4.805681457561184, -4.959588437581088, -5.013259465745109, -5.070148436368585, -5.778302005668334, -3.0289910877331776, -5.527143864017425, -6.11105402601701, -6.123887370951674, -5.975929983744964, -5.442482986164537, -6.151079378646925, -6.120416006196215, -5.065655600056955, -7.071974876907828, -5.957253719340739, -4.781773843113505, -5.354948164056644, -6.520450343119245, -6.293473674919533, -5.817016517849025, -6.85914069525248, -6.81356318675616, -6.777584585720929, -5.97764525003172, -6.226877237430227, -6.02961643394924, -5.9729353291005545, -6.418715298488868, -5.532354351604477, -6.803710890313149, -6.889653320113874, -7.354541848692839, -7.502757793000282, -6.938924369120657, -6.214291571987144, -6.749334119435077, -7.237514956739779, -7.394443065445024, -6.457482478746571, -7.947443614261728, -6.86121538479334, -5.698583920623846, -6.845759126556648, -7.683896617545187, -7.180188461548061, -6.757725777071325, -7.8704825731256, -7.881944016644606, -7.107203568791572], [-6.7847523397462, -4.956382101946025, -5.55168468117927, -6.54694976738503, -6.679027997805541, -6.920449307304025, -6.651631877885628, -7.028532297007316, -6.819873006155468, -6.9668156836069475, -5.07824502580267, -3.6978375281608122, -3.2158001218369128, -3.221520357616586, -3.5888259593254497, -4.18118917224718, -3.0226572253435795, -3.971027081545084, -3.562365839815367, -6.241860395850777, -3.738063429007306, -2.286143322088719, -2.8154088123469325, -1.9366972011829824, -6.402763696295053, -4.09673956284193, -6.7632686126933175, -2.217315348390325, -2.7898081670861736, -2.2970370063901697, -2.7943042308650328, -4.45525252224197, -5.6170558845712195, -5.634999368405547, -4.204524838877218, -5.247540598697068, -7.679731693977602, -7.496500850432476, -7.694782629326121, -7.660949127015419, -8.444625153562612, -8.02242837580043, -7.9107342568656245, -7.844229700045118, -8.496568431723452, -8.90528981102539, -7.744710798608082, -7.796294560812227, -7.4084137546993, -8.54315037261458, -8.96995880381818, -7.702981796781846, -9.818649422090688, -7.982756374523151, -7.087136791691802, -7.726484291615801, -9.302472289566692, -8.264363329731667, -8.520253749464962, -9.573314897467998, -10.041792973404897, -9.03352354731742], [-5.530908158078372, -5.12660572425173, -5.285205739999349, -5.724471900365845, -5.873312125055024, -6.16534879225732, -6.148934708170209, -6.3610443530793574, -6.311033932504697, -6.085846837289822, -1.9318371570117603, -3.9897678769234868, -4.264721024425275, -4.508665668126993, -1.397989588361542, -5.165043688633805, -5.1354703161064155, -4.710076673495516, -2.2812710196631696, -5.744808587095395, -6.153180999051661, -2.7741841902487896, -4.878049696777151, -5.0409257316634655, -2.3898944653854226, -5.13595610674021, -7.841822106027579, -2.8471437524600627, -3.4340095911141377, -4.619316513108799, -2.3549200090866647, -5.921518406898233, -5.302848234969303, -6.786138406296854, -4.207750888408638, -6.039012800612939, -7.841822106027579, -7.704962923310382, -7.065002449743519, -7.589296290560376, -8.260096172428325, -8.035112694192108, -7.871344545293901, -7.140827013169332, -8.545938317958099, -8.836893715754849, -7.993349965830133, -7.27788665694764, -7.741565502287929, -7.888977243944857, -8.111485672976682, -7.875096894912451, -10.091297373707675, -7.392485113590403, -6.863268166852988, -7.610229500226842, -8.901213681542407, -8.92241588919301, -8.313038591698524, -9.803615301255896, -9.310181420201774, -9.00812940746704], [-5.262771475279199, -4.784242541566078, -5.27970172852834, -5.684918512820975, -5.945292208575153, -6.218979090786592, -6.228233983716081, -6.354509212158639, -6.204315840281487, -6.442281485719593, -2.526983467749325, -5.114296740462473, -4.530006859098648, -5.422625873895182, -2.9651924820539826, -5.615416770210166, -5.759915004501759, -0.738654569892248, -3.9951800832331146, -7.496424744983864, -2.271649654828947, -3.6688863968618732, -4.479075949205008, -6.406971077861438, -2.7704548687771466, -5.064362048924221, -7.65209604855975, -4.10990241837356, -4.3379100438872245, -3.144791234860585, -4.257400003418622, -6.678114421469913, -6.866487433947222, -7.697728752690109, -5.8045538215356425, -6.443871311173693, -8.079096309219214, -8.153204281372934, -8.17767713789166, -8.353264662504271, -8.864090286270262, -8.820319623214933, -8.693771874337664, -8.686253041923635, -8.948055666387596, -9.842684901480572, -8.909864867655205, -8.569719225667685, -8.285794202093616, -9.08611589448313, -9.094531091408415, -7.571541334321345, -10.622843459030147, -8.818180584966184, -7.823031278179709, -8.265917771336019, -8.826764328657575, -9.481851947393176, -9.0917181500318, -8.558120089824332, -9.233307603986084, -10.138335143581529], [-6.352576341887475, -4.375482811098582, -5.097247890129581, -5.253867383491415, -5.300921501088017, -5.719261065235271, -5.695077377144448, -4.664604980788103, -6.0509261540753565, -6.1211148128546835, -2.6955840583648807, -4.0885439717448415, -4.601612684872268, -4.3272866151564005, -0.8335121453767717, -4.368698782518719, -4.841722524158204, -3.6648788033138118, -2.5172646408008483, -5.8838060715240985, -5.03345350892465, -4.275021730152804, -4.6405776530763765, -5.25643229403688, -3.133213396702524, -3.881602832644344, -7.333792367288475, -3.6375800962202542, -3.280343679384232, -4.168603006615407, -3.0158762344692915, -4.957870530657444, -4.673357005846336, -7.071279326505898, -4.732292586433726, -6.102987428262127, -6.308187629826426, -6.692855118141101, -6.912844895031934, -6.811275201992611, -7.387021770702892, -6.866694309094154, -6.971677700053924, -7.120054614261141, -7.672123196561181, -8.47105299909571, -7.06564216575394, -7.303313817631059, -6.877175818825198, -7.588372692026248, -7.710844848156796, -7.012073355807918, -9.487490679574803, -7.203069557208429, -6.086293297912648, -6.450936411500557, -8.098699438256325, -7.677018501414594, -7.313496820953744, -9.611543328244782, -8.931691690166081, -8.112931574673333], [-6.278777370845214, -4.701340011999448, -5.078865738235898, -5.8830885176004895, -6.165050673343658, -6.274964098411205, -6.212416394644692, -6.400778791567437, -6.741692863598321, -6.816576094912807, -3.3928904712107735, -3.566247086795751, -3.646180662937257, -3.523693462975528, -4.243767227003855, -4.080529251966207, -3.795135190780714, -3.596123418433819, -2.7581828008765474, -6.336206897605354, -4.412794119233608, -2.7790238541384125, -3.7937492847610685, -1.8714655708089012, -4.039189587014222, -3.917817104421266, -7.57147815542886, -1.7211826197805813, -2.454732555101985, -3.258178620019967, -3.831607640543702, -4.745787338078659, -3.8173867609921763, -3.6132042458306612, -5.788388393117924, -5.078865738235898, -6.509442541403006, -6.358094961721853, -6.982655840165303, -5.955633264954192, -7.203135532877834, -6.519814112808369, -6.508748303549079, -6.5805671825510945, -7.521390752829464, -8.307422019239775, -6.987204748691672, -6.90870144071007, -6.578332663638785, -7.263487211998134, -7.281252691733005, -6.635300404511988, -9.747550691415777, -6.939413018359865, -5.8716986926591, -6.717215468733563, -8.061948046761787, -7.4055504084105825, -7.052394044113555, -9.471951436870118, -9.440772718621666, -8.392393495069179], [-6.019277337678054, -4.809610930457335, -5.353723551448762, -5.676382755872694, -5.70371603264992, -5.986064759126338, -5.8139317591377235, -5.978144349530662, -6.042881895447475, -6.198360420377549, -2.2921047448703056, -4.34095023979577, -4.345648109454055, -4.367598309292761, -2.0155741702111323, -3.0132780221373907, -4.923111941566577, -4.563679111444075, -2.223531060167143, -6.691102686207805, -5.506010794027101, -3.0594865563826104, -4.849221407410736, -4.504665385487454, -1.9294270482862954, -4.87704758753489, -7.8125932274537675, -2.476262613634102, -3.9821539473427885, -2.8193399945915623, -2.274864811699418, -5.85738068573614, -5.424700477670406, -7.189965927648669, -6.389275800912233, -5.638934468614081, -7.03745392086323, -7.341255375194895, -6.260863632640313, -7.279063354938447, -7.679061834829245, -7.811259004440631, -7.236144890792278, -7.5275849533863886, -7.494018129747305, -9.1181262319292, -7.907769904821474, -7.576977708715964, -7.061731490444577, -8.083943001314976, -8.105182737815888, -7.709906510180343, -8.65367988774833, -7.560240916360441, -6.676421938949557, -6.335428678009885, -9.006382193489584, -8.143473650809202, -7.5376758563683515, -10.154666091954931, -9.349927845986523, -8.656780665426579], [-6.45948206141902, -4.541389017548509, -5.0469147172405115, -5.714893702071352, -5.93765911006474, -6.159588168280474, -6.484829040506417, -6.54643138634695, -6.777423978144398, -6.472536180151478, -2.0992585177991696, -4.351360197286945, -4.810779680772453, -4.478347221038041, -1.057979923268165, -4.503803429976553, -4.369600328148692, -3.1761274441594316, -2.5925706700973334, -6.132316774171815, -5.0527495258256305, -3.548319221739584, -4.665298532220545, -4.239092161682817, -2.8823289365699227, -3.9799558061803935, -7.613527691788449, -2.9149052754992715, -3.0856952412535055, -3.988705873974733, -3.519423658731669, -5.380012776918168, -5.236242140888087, -7.294418265937878, -5.378236234528395, -5.653900468202214, -6.843339306558724, -6.1506932535462235, -7.23734725322546, -6.366553451469326, -7.364460455464448, -6.868708500684183, -6.652923233094938, -6.285920796794185, -7.434940854111734, -8.332992180665768, -7.045904469629325, -7.3935637933916185, -6.8666551118723405, -7.771896007559911, -7.6868589648739984, -6.982133113853507, -9.79784886428876, -7.098382406877942, -6.05045863483477, -6.643853499354418, -7.8988076492249855, -7.62952803313489, -7.2191872449632415, -9.030593711575094, -9.78510983851133, -8.512650620040239], [-7.312010848930209, -5.9704021836204655, -6.3878250976208735, -6.964626238636961, -6.946983784483292, -7.4124339237241115, -7.3181741966378775, -7.680384414313297, -7.712878247789743, -7.554150900935073, -2.446039713635401, -5.2426041270797, -5.947299500896444, -4.663409565765878, -1.9484553212571183, -5.736061416021277, -5.712018325729332, -4.9933347157551395, -3.124667578606162, -6.881965531986077, -5.837272445819192, -3.4071965596735665, -4.466780007835062, -3.7439634287862087, -2.4245394005655605, -2.7481043832730636, -7.96925132102691, -3.001717011027925, -4.3613498567188325, -1.0557054893757776, -3.70597885137566, -6.306264213716016, -4.571775536818785, -8.174476930790583, -5.896948417633875, -6.225037976827956, -7.408704473732679, -7.482057817244962, -7.278478028822127, -7.6772808871048746, -8.27951403118039, -7.874318428531343, -7.723948471543991, -7.700568997316141, -8.91379761462879, -8.048158534515101, -7.944081716081054, -7.980577820654186, -7.290733450482923, -8.159308709317411, -8.547715834521, -7.821812931153412, -10.179602578494553, -7.600205085285631, -6.935111692528849, -7.541317747952087, -8.80008790436004, -8.364689804291782, -8.08874001915325, -10.681459532277783, -10.11675467483489, -9.167510387022356], [-7.888212115175222, -6.644858293408979, -6.899926832152373, -7.644887076927795, -7.6845832447435445, -7.532598742461448, -8.35541557609491, -8.047237610771814, -8.039272966898704, -8.151069111852731, -3.668932909455333, -4.782855975285661, -2.6659061839419613, -2.8967154778709228, -2.0223904901109755, -4.446771913881909, -3.000219123980515, -4.8870757508472185, -5.930891109758962, -6.823383785692898, -3.2455278628878323, -2.6715253337166462, -3.4540042231373063, -1.613293110190054, -3.32378892967727, -4.249589417392827, -7.22956773062339, -3.479217249500404, -2.6394006861426296, -2.505331420610824, -5.608431497999796, -3.9128781321838506, -6.817651617834854, -5.899129031817539, -6.838124773860243, -4.798082958969038, -6.707198867504101, -6.862144714309829, -6.946702102337892, -6.119321735858142, -7.3106242225785305, -7.047703884629593, -7.1735103563413185, -7.1142937780684905, -7.8727953781736435, -7.684274173500164, -7.105593531784784, -7.043298950514618, -6.705339482889372, -7.153500510194026, -7.545964351744817, -6.429130146514012, -9.208169648866278, -7.146978747947639, -6.381569697965211, -6.863775814304458, -8.274721978073412, -7.87653370028425, -7.678419897035876, -9.611509357662127, -9.195484489338961, -8.64028017837942], [-5.479179331884548, -5.058052917950845, -5.462604366790335, -6.042749073104725, -5.955329948912708, -5.958323963125313, -6.182592584875092, -6.134964535885837, -5.987223312688662, -6.092926822690444, -1.4103668710151773, -5.3350267357418115, -4.970584606980225, -5.289457186272422, -1.770010358993832, -4.901554513033236, -5.37688758694302, -5.089624713826659, -3.3520314691389803, -5.392942008652894, -4.734460344193294, -5.4364649785873, -5.373540303482963, -5.098468738623112, -1.787587681664556, -3.2734283276295595, -6.018573842572738, -5.287155689284143, -2.4508369623232906, -5.268931950327691, -2.1042922431924556, -5.627005270227098, -5.60465202444635, -5.974953220096848, -5.917202163476144, -5.89303609762896, -7.4048682036926285, -7.92851451588768, -7.027392005486696, -7.609083745121319, -7.38594019380711, -7.373517673808553, -7.797674914928869, -7.69894007424318, -6.9721762828842495, -7.92851451588768, -7.9072371174403955, -7.498269378781029, -7.24617825143195, -7.836141195756666, -7.972477639308797, -8.030297210197622, -7.424161406627308, -8.130380668754604, -5.985681292336846, -7.548459123304884, -7.876146530370365, -7.886403030537553, -8.383576565135217, -8.509870290459508, -8.104405182351345, -7.3612475812167375], [-4.615704248273826, -5.059874046144977, -5.605352624766998, -6.125932548297868, -6.262849675716521, -6.79239877505316, -6.990929985784472, -6.841047905444769, -7.1584533214794765, -6.9886390340379165, -2.1061188583709898, -4.694960766195293, -5.294767434713961, -4.877455498156362, -1.7372915500183557, -4.54913156517679, -5.204152324138941, -4.129384627657894, -2.037945671503165, -6.073264053143508, -4.589028985169175, -2.990753289694604, -4.778969963958759, -4.689721987339014, -2.425686266522293, -4.746481928875957, -7.836127606736178, -3.351158263461062, -3.2418172438356554, -2.1018059989954563, -2.7036646563447677, -4.616611132654885, -4.89196974360223, -7.672674534246606, -4.594182160042578, -5.702759391924034, -6.928687676907251, -6.888038111045066, -6.879281938695605, -7.0247524516915965, -7.464610089759517, -7.314413547152238, -7.29014621148154, -7.293244587314209, -7.623992355607218, -8.372647679876952, -7.108856380340105, -7.38925322665568, -6.931385277679899, -7.915280634635728, -8.074744053353632, -6.836124212582984, -10.164407149105006, -7.372318020898097, -6.283267109907934, -7.0071153102054895, -7.702596214111655, -7.848216404055182, -7.332806708624255, -9.549591811814969, -9.295811291038868, -8.128717117098297], [-6.86596902684244, -5.052082122851932, -5.150933488599681, -5.292208274141252, -6.214682697483703, -6.745331904339627, -6.68552490120206, -6.760045968476806, -7.227305715576431, -6.950863382929711, -2.1215717821722215, -3.9228106947364623, -4.840908762978201, -3.0664686008437925, -1.6983460154315366, -4.061987675375524, -4.329017422576037, -4.468146834761754, -2.0559567932823986, -6.284656471404593, -4.495653917304999, -2.239702601986903, -4.059167724800704, -4.675038779649125, -2.554078005507218, -4.208324039380938, -7.846798279854166, -4.952711862629422, -3.1510988852700317, -2.942373372199169, -3.3895078376634635, -4.604176621456193, -5.1138376420493366, -7.9144140567925305, -4.640693910453494, -5.102920313867999, -6.555881088992486, -6.826369644359554, -6.799686147757116, -6.863359893400418, -7.558312671802701, -7.067443904093005, -7.193279183729173, -7.1899393531824245, -7.705592764618944, -8.457679157012771, -7.256276285899237, -7.186610639983997, -7.0503828029559665, -7.591798854832744, -8.06246715675167, -7.178062400973902, -9.768878866076495, -6.917516203514602, -5.665418470446385, -7.260143611855673, -6.937390775844807, -7.687607497719584, -7.3485921408206325, -9.2384661090352, -9.464847655623185, -7.897922355187247], [-6.774920173861653, -5.204290861824316, -5.43566671911157, -6.296930641076284, -6.565626027570306, -6.7373590105822805, -6.991252066666452, -7.063233360250443, -7.2676834795551315, -7.1966044322333405, -2.083909994196951, -4.067585265323549, -5.182321460673428, -4.835732283228186, -1.980079058156674, -5.0283663440075825, -5.072641431371989, -4.6392056435133515, -2.7086094993416667, -6.770304135451181, -5.008181972697398, -1.1019554288598505, -3.2594369265598493, -5.6765883311043766, -2.736784393019736, -3.364824539297133, -7.918041582638827, -5.789527217909832, -3.4282422713486094, -4.498435097687583, -3.6322465613937016, -5.355734234640129, -5.0804532195051415, -7.17597860187687, -4.9759313787901425, -6.718299350460867, -6.452973839496266, -6.266241964550636, -6.756169624516277, -6.43275589071587, -7.149632742116933, -6.722481373881418, -6.632646871978412, -6.5830057991087605, -7.441299494170494, -7.5325638154552745, -7.295254585596671, -7.196283868037615, -6.683920964952739, -7.4835386813502325, -7.587428922371031, -6.992558402580507, -9.54477866233574, -7.0691430439756715, -6.0008086042221205, -6.0079129112663265, -8.273962947039259, -7.457398913104675, -7.183861348039058, -9.235518989239411, -9.247910721534573, -8.527701618931673], [-6.725963719695806, -4.710012269685353, -4.981543035572536, -5.7693914443296395, -5.741238652017211, -6.34104225410854, -6.461701921452123, -6.616580803658851, -7.050213688876461, -6.932988782385651, -2.7442254377569215, -3.902092293780571, -4.081087684032349, -1.8459531867558991, -2.260196124051401, -3.8113055093318327, -2.1904142607255017, -3.5077174805325915, -3.070625904070985, -5.688404814159279, -3.676818033328958, -3.705969932950152, -4.175966474127823, -3.555777846153588, -3.6599169276373167, -4.233380646142029, -7.565625193952616, -4.758247211588438, -2.668505149650637, -2.593317887541856, -3.982238613532973, -4.922572014698711, -4.457568202222798, -6.922768742308341, -4.874097132395777, -3.7366670217502587, -6.099051002533345, -5.950541812233132, -6.754321355464054, -6.000835251677606, -6.818602573551164, -6.538250446625281, -6.5136752899401404, -5.7389354920585625, -6.837502673710269, -7.922940958163197, -6.4879709082904045, -6.559235554477551, -6.415308634074, -6.963830229106114, -7.464928001069955, -6.466545899251641, -9.489635645899149, -6.3635319575989815, -5.527598474760337, -6.407003965795449, -7.463914829757348, -7.418626873140783, -6.4607730713017455, -8.850715648219035, -9.544876513609825, -7.984492491801492], [-7.251231594323855, -5.21374696908488, -5.698223064924641, -6.1375976604763816, -6.322511629217009, -6.768362468268195, -6.494106816635844, -6.457670861321346, -6.860741199932812, -6.755693520691695, -4.0580333694338515, -3.9411669781297105, -3.7467312772977017, -3.0227837077197126, -3.61529958808048, -3.574469853485688, -3.2007714947116837, -4.080705573827712, -4.670471138732819, -5.7966968651863855, -4.155602316322404, -2.9527040483599185, -2.862751521261682, -2.037725742388079, -3.7818488691194445, -3.113228390392548, -8.164264994327704, -1.8458475827404217, -2.5467378386179824, -2.7020967605017185, -3.188124830525815, -3.6799312814828857, -3.4909044267832696, -5.197365837878935, -5.202188998354364, -5.870321468395694, -7.4721345906728445, -7.508740887837122, -7.308294052687071, -7.524695844262695, -8.528908107915612, -7.92499206108361, -7.684820971302881, -8.246116020758024, -8.52404191826444, -9.16793672064164, -7.835273241415741, -7.178620119286367, -7.559608480967977, -8.499112535221915, -8.875184344633446, -7.775193695995215, -10.198665810583236, -8.01769231637253, -6.730988787795164, -7.673144441641045, -9.323628945401888, -8.330457169191774, -8.013022138590646, -9.77099654516406, -9.908396414566504, -9.416350685452109], [-6.383306695601549, -4.903894061372386, -5.294037540593016, -5.450500954330478, -5.333223053427145, -6.308171725769449, -6.710407117707228, -6.887542628987831, -6.847572675445971, -7.035457528925254, -2.2105034037003235, -5.0202198363584465, -4.228806005807516, -4.2610399361561155, -2.630714925229401, -3.899363165669206, -4.911168632565807, -1.9598915311908123, -2.357698156549484, -6.874278408705108, -5.4856719078268386, -2.9453898821076563, -4.76865138002776, -5.849682782165729, -2.1582691520417923, -3.13711767038794, -7.932068702852963, -2.1341546758789423, -3.3759594869296796, -3.525124340216131, -3.4812218611704324, -5.083766155674364, -5.7936328268294535, -4.353576622920111, -5.907706290260866, -6.135534326871947, -6.846532994187875, -4.609063889576182, -7.503287789431182, -7.483445868632177, -7.2287173521187755, -7.853210127940644, -7.155334806184752, -7.407188463381393, -8.090673733029602, -9.081164896953064, -7.857960730699241, -7.896800564015505, -7.262268886290009, -8.128645226782451, -8.177248142566354, -7.485412437604219, -9.720755973044048, -7.9739596445621235, -6.796539470357211, -6.907958941482774, -8.646989682962468, -8.283628546124925, -7.797896489956591, -6.6686665609139695, -9.771081056931564, -8.750720965163202], [-4.685181991517679, -3.1394221383869034, -4.505733999919456, -4.7128201143378075, -4.721709061755053, -4.800636320344724, -4.867216645426291, -4.85864761270119, -4.820149134568306, -4.881081685563463, -3.6610088191091923, -4.523883318425133, -4.405422178721609, -4.511747247142908, -4.4973757423052545, -4.410859930021738, -4.692381446660533, -3.970040309575452, -4.061847858828575, -4.724689689893191, -4.678033998252392, -4.350556098635668, -4.167208374486401, -4.450909776342023, -4.644417387453407, -3.724277324458299, -4.630737283549326, -4.509337607422754, -3.787718752781497, -4.309323510802315, -0.6823718671879156, -4.449775346271411, -4.306372205167735, -4.410859930021738, -4.920232593125288, -4.3914208729895154, -6.478377211171838, -6.487035273914953, -6.801150603434889, -6.531487036485786, -6.875258575588611, -6.039010551387992, -6.9553012832621475, -5.362670489144429, -7.042312660251777, -7.027274782887236, -6.983472160228843, -6.58757650313683, -6.732157731947938, -5.820321350423163, -6.504579583565862, -6.710178825229162, -6.395685495326725, -6.710178825229162, -6.559138567816296, -5.802621773323762, -6.789316145787886, -6.8131267944816045, -6.29749346914751, -7.1889161344436525, -7.012459697102096, -6.888161980424519], [-6.973841065843466, -5.090695336189662, -5.341561221116127, -6.0115634498929555, -6.202945132409144, -6.628849553394376, -6.911669409716161, -7.0009992844957765, -6.886081347969349, -6.711294966663227, -2.260331825727248, -3.839334077735597, -3.9667628422059718, -3.420241689905642, -1.8735787833427253, -4.188499103605226, -3.691171678101274, -3.8593630820739957, -2.3524981891706576, -6.231341006564324, -3.7662373591100775, -3.6465568924129386, -3.4963200738799767, -3.4321214183238204, -2.593050419274572, -4.346868167488547, -7.8364379197408365, -3.990422275679548, -2.904818352752614, -2.3714612152432646, -3.1378637759089556, -4.357037863124861, -4.651314292504279, -6.760867254793699, -4.107313187039242, -4.3976920060255535, -6.440759053046417, -6.465477098310146, -6.582182413654962, -6.610599862587676, -6.978263755093254, -6.719449335464573, -6.730879435939417, -6.514035039942995, -7.843934206718764, -7.66305587798569, -6.780807453646412, -6.810345466083276, -6.347157031161333, -7.321885120582923, -7.779113558339937, -6.602957069625781, -9.658525717989464, -6.893034874465941, -5.640541915214837, -6.61808064554899, -7.825297811959789, -7.623352248005446, -6.729273803531229, -8.625119353004427, -9.394209194493895, -8.033505455437194], [-5.604855175474509, -4.462285490693087, -4.571350651692403, -5.056493889833258, -5.072440143885144, -5.681064638387167, -6.280536665394942, -5.74786868753166, -6.300402104581103, -6.443541857970619, -3.08781816164275, -3.6971443171127936, -2.396573090959646, -4.216785762291264, -2.055069601332964, -4.2400357108180975, -4.510301966657801, -2.8725223515174108, -2.985980079879496, -6.151031964062443, -4.17853136290412, -3.739735408911567, -4.177743865892257, -4.758224336321508, -3.406169247786832, -2.7407038589797272, -6.9336687884910315, -4.884332887441078, -2.4403221799860293, -1.8263508274951405, -3.4054696763623764, -4.562592776233452, -4.420217843796468, -6.782722037918705, -4.9372619598181355, -5.457485531187882, -6.256287610141627, -6.611108985431037, -6.565907361549403, -6.49534726380481, -7.061567128112425, -6.599672447239019, -6.7013251912547664, -6.80746927442344, -7.516987128232732, -8.06324629910308, -6.809992050813993, -6.860502751231873, -6.646463246903641, -7.480934599971211, -7.54322475844959, -6.575185095427639, -8.985698479921282, -6.994899040702809, -5.767168821265772, -6.530694737719788, -7.839611467995152, -7.435063610819411, -7.091053743823944, -8.707954377459634, -9.10781723001913, -8.370947238445352], [-6.890344120648169, -5.02940421086846, -5.408867487897182, -5.967857793669728, -6.047653312106489, -6.46237897058548, -6.749954722267399, -6.888388607925276, -6.98955227182629, -7.21719797180092, -2.4868466816371, -4.771074471877903, -3.6481121610473157, -4.664656454469179, -1.6817234068168874, -4.586866350603287, -4.845552155508254, -3.154508356873059, -2.57905575846229, -6.254276196451527, -4.921506816489567, -4.4705470709445265, -1.6402347840968394, -5.1280286653720175, -2.882844123251368, -4.595878517780293, -8.231899529453244, -3.131328577959143, -2.841674626039629, -3.3059284308174086, -3.1776616460738145, -4.478455963747706, -4.881954395151054, -7.651052083171771, -4.569886902888001, -4.27044926534142, -6.859624364774909, -6.620591583748609, -7.035724951332805, -7.0226528697654516, -7.663432064808081, -6.880725735179707, -7.096836038369936, -7.001238775261635, -7.94609099384926, -8.742881147291776, -7.2506609077408095, -7.3153280695559095, -7.053568548019118, -7.728589206441718, -8.033869830584925, -6.95515998225557, -10.250083546611185, -7.317576103473792, -6.34314386192865, -6.83930777142871, -8.443181097280567, -7.8493114938202, -7.476614929901965, -9.60010853791645, -8.54213954241637, -8.677952686589895], [-7.97530600380511, -6.415691397743754, -6.861539964259856, -7.354178676035024, -7.5544322703441, -7.956843940965374, -8.00285595557334, -7.941715059369074, -7.997284910523884, -7.042078762400056, -4.593656623473984, -3.4358427231129607, -2.813804032113097, -3.8443096208464707, -2.253236786554918, -3.523372060796657, -3.771274119941123, -4.213005093082092, -4.26676474950711, -6.604661414352346, -4.071781743539046, -3.0636392294004997, -2.748766550188664, -1.5979085155684218, -6.096125476162904, -3.8319741165296572, -8.437883856353995, -2.362166563210972, -2.079080404566457, -2.423752746988421, -6.1373971409737065, -6.112261605870961, -6.033619491127531, -6.187046440930159, -5.880551407088361, -5.508879098944167, -8.771401153617497, -8.582218506443597, -8.081702252336468, -7.820268405959624, -9.01805471242806, -8.991966276343762, -8.037776271878622, -8.339328192194554, -9.20125771484982, -9.932145223392613, -8.711282886564021, -8.633361864553784, -8.75433589306331, -8.724333642759511, -9.49070501282617, -8.37573721938598, -10.194509487860104, -8.552742688310644, -7.968342726152964, -7.956843940965374, -9.863152351905661, -9.598817643450872, -9.08877973142313, -9.278218755985948, -10.729432663205156, -9.20125771484982], [-5.1527222081970026, -5.012025127950751, -5.39388426501389, -6.217687012129963, -6.355002880531065, -5.6351154829069525, -6.363993950090922, -6.903492332891861, -6.853964796193012, -7.129436704737862, -2.71361594954537, -4.917813116205636, -4.909342477322127, -4.834757647401019, -1.3606202815267094, -5.049186428683174, -4.831934627067998, -4.02878090419557, -0.9857025802248848, -6.9583248115172545, -5.928056254423655, -5.1548164498001166, -5.532591229650075, -5.575217341325883, -2.1159432194090155, -4.2897086796604516, -7.362881241689201, -5.200780422733714, -4.007732553855012, -4.899317914996637, -5.732412626945404, -5.409409253401815, -4.570479874572386, -6.004563148213606, -4.806034206660294, -6.602559371961363, -8.008146936015933, -8.264296783798311, -8.114951495245993, -7.067733915701082, -8.653122572902513, -8.40456220860418, -8.35558820400772, -8.362968311305343, -9.101577865942044, -9.242891399008622, -8.693532111240389, -8.468328826963168, -8.348262163915647, -8.71436619814323, -8.123659823137777, -7.872345394856871, -9.1093600063841, -8.693532111240389, -7.279281672853908, -7.536521246398329, -8.924437667890087, -8.99854564004381, -8.061492916721225, -8.91796515338447, -7.579574252897618, -8.90514446495541], [-6.625065979493934, -4.788580474517243, -5.256407824340383, -5.37832984858721, -5.2038978790684185, -6.079823689638168, -6.317086566978479, -6.646320764762257, -6.72855886299923, -6.886116701148103, -1.505097000684665, -4.82588581184553, -5.253602398681012, -4.989577089424305, -1.5758741362397704, -5.185784176269271, -4.916501452893276, -3.8159863951228723, -1.6041923545980379, -5.633841418592661, -5.7199839934992545, -5.2161585690547705, -5.369003667873634, -3.3721189713516244, -2.661522576130661, -4.670108701549678, -7.227790153984301, -5.183916064386551, -2.494756625753895, -3.869077211017653, -4.27104038948766, -6.0954898063825675, -4.71167529774246, -7.136943681210728, -5.987593808088792, -6.590446143960144, -7.403791666759668, -6.814987319540992, -7.030531087602099, -7.451894781778061, -8.262224216961, -7.8389014914528286, -7.199379152151521, -7.871278420177289, -7.002535480112671, -8.630549778119708, -8.06204504276704, -7.169938203345767, -7.199379152151521, -8.169374062997537, -8.264930578558742, -7.090662928646722, -9.550754409315, -8.033689817011913, -7.078178371984477, -7.674437552058881, -7.921898411023797, -8.116604027017473, -8.273093889197904, -9.39660372948774, -7.831846788472939, -9.063739434242727], [-5.33959692367457, -3.901875433911095, -3.3263523840890916, -4.682859878096868, -5.0569946532038434, -4.833678156338227, -5.439237690659609, -5.4278954140556746, -5.662170026427569, -5.462315637942154, -3.751806551579704, -4.660118258928943, -3.8333548860836233, -3.8811153084238335, -3.4958051088141735, -4.553750303632283, -4.534932311997922, -1.8458046740483725, -3.0281790617647277, -4.436160927561686, -4.943995470762378, -4.549402848736927, -3.9218705969950047, -5.328568099202016, -4.452091297246621, -1.4302390343932077, -6.080922600826194, -5.1148485422425125, -3.3219189169449392, -2.621905648275213, -4.3789508674983555, -4.262632175345694, -5.088920047216344, -3.3849839728869298, -3.307281062205608, -5.974439120423745, -4.379829216259262, -6.791884017661267, -6.8166991867809905, -6.068955964208674, -7.1565271312491765, -6.683586642330131, -6.801736314104279, -6.532793078538745, -7.518256845849533, -8.152370494825284, -7.391505140210389, -6.7194113688482755, -6.416271872702784, -7.279027156783699, -6.4871905993829495, -6.66039719741122, -8.204669994228134, -6.878895394650897, -6.094660190607558, -6.427558674237414, -8.084652320770335, -7.339319387039819, -7.347830076707727, -6.950948712290955, -8.303341521735163, -7.468823387994359], [-5.431835629645527, -3.900968951139421, -4.151004765492447, -4.646528675532386, -4.521804796114552, -5.283128043243618, -5.29007437093391, -5.66335521876192, -5.740643416211578, -5.366660147370001, -2.670098748915231, -3.286242237340954, -3.3585141055181613, -3.755602707715123, -2.5586532961090027, -3.6926656242369105, -3.8330192649391925, -3.254475467118251, -3.8290307266838655, -5.019290301102854, -4.363753678748955, -2.9684754808494547, -3.151618081678543, -3.50657051212598, -2.895083629045828, -2.91665360144011, -6.285849431998168, -3.6228229122699127, -2.152751777383897, -3.3162167836347605, -4.2488413058475105, -4.652746019284857, -3.9288228848114928, -5.349158624766704, -5.572231075081914, -5.261791793324819, -5.624778734167247, -5.278347157643276, -5.3043854022704995, -5.210237912616575, -6.346428699255328, -5.93048158286466, -5.615084400228674, -5.89255288484716, -5.916841563358977, -6.777270925569916, -6.237681480853062, -5.483828321441287, -5.599254877723378, -5.735607736245962, -6.332645938959907, -5.443956990177872, -7.787390135984574, -5.974679054270741, -4.487700164671342, -5.484479363130949, -7.787390135984574, -6.624646575217306, -5.915838554247808, -7.256570295689133, -7.872330812633658, -7.3300889926342885], [-5.8962959205599565, -5.022482812723318, -5.262967235210922, -5.638331471894197, -5.778457744433211, -6.3109845765456285, -6.29163873510587, -6.56832765910669, -6.49255510063436, -6.3650100589758205, -2.863492599093995, -4.089024629780227, -5.154949557468251, -4.484159695248834, -0.9343644420039627, -4.498161263246394, -4.6279862283262885, -3.982370058053253, -2.2035282537140866, -6.199482179547703, -4.571665803362194, -4.084960905072187, -4.755413554835734, -4.800997636581884, -3.6971982708382995, -4.566795129611478, -6.871351961660363, -5.099526707333125, -4.135788994836005, -3.1706024468180507, -2.212641008744242, -5.078715683958774, -3.5811841558979225, -6.741940888008861, -4.856098125468122, -3.980907107020626, -6.712325967469931, -6.644006723492454, -7.153784678226001, -6.945055157447498, -6.898020208742524, -7.021563043483541, -7.173168545047049, -6.854391346353318, -7.669859657878135, -8.284314569862415, -6.895321151773359, -7.513205847832758, -6.795366054682441, -7.556635405760094, -7.827916592672458, -7.241272132349116, -8.35137880044296, -7.187505708193456, -6.1188193168155305, -7.139919638088829, -8.075324765986299, -7.379987004046535, -7.364785842714484, -8.480789874094464, -8.527936652520165, -7.717773013574032], [-5.1596391409467905, -4.446625879300866, -4.792661465824471, -5.092676376197713, -3.486026851337387, -5.293817992318437, -5.4207097441730605, -5.031624556083574, -5.433394903700377, -5.123410202650332, -4.715036303793935, -3.7021536789683007, -3.7870302329121537, -4.088809455118969, -4.812132309601868, -4.898021981482298, -4.704305701448848, -5.045386241156256, -4.694501701352228, -7.439823297251669, -3.3979941838940024, -2.7387677229303073, -3.4751598040428164, -2.0375451283648167, -7.179408351140228, -3.811103778633784, -5.971540702552751, -2.9729698958508197, -3.825149191015764, -4.478865937874288, -2.6641628726801234, -5.3193066285353545, -5.996634270150407, -6.375549146164451, -6.785390628872414, -6.572829186888741, -4.124487206195906, -4.535138151336436, -3.7135200209337587, -3.96463850525401, -4.554386339641352, -4.765674647825141, -4.429638997109137, -5.277664924510665, -5.02086262836912, -5.937153360243275, -4.7565501004863675, -3.693750268171921, -3.749678820831146, -3.176252392796863, -5.722496422806384, -4.546642548744449, -5.417354036326088, -3.1803577086920454, -3.749362815140414, -4.066652137558128, -4.660532401294013, -5.1570534898617675, -5.430844966067104, -5.284241491875228, -5.5748604240003194, -5.63236491848051], [-2.1543734650260515, -4.349659150265929, -4.518901599180162, -4.334150339570116, -5.146931928181808, -4.726677895862385, -3.8738862022041083, -5.547738126032732, -4.965611539500847, -5.329386916158828, -2.0510193107742145, -7.017067715762613, -6.5514024881154755, -6.782731624016124, -2.1387030138865364, -6.887331394106641, -7.26098839501858, -5.915573580719614, -2.792353319034718, -6.475549145597215, -8.196805549243091, -2.7290425937620233, -6.57338139483425, -7.482751683863832, -2.7069370493135643, -6.6217549791184664, -7.959675755954141, -2.6929063910157374, -6.402701413055781, -7.378081061299943, -3.038722067403394, -7.146279447242619, -7.442205589469482, -7.777354199160186, -5.436617340201765, -7.9268859331311505, -4.21410095816366, -3.7159537926574395, -3.274052690014408, -3.8709022387814955, -3.903841048493916, -4.907248138723605, -5.564815386850924, -5.634025390029099, -4.794669640998671, -6.37044055083756, -5.773624479216047, -4.984713243886536, -4.705003878814023, -5.884409585684786, -4.869643914101749, -5.8105008762596295, -6.426399204485604, -4.552569937605819, -4.688840192155229, -5.1496039404596194, -5.387063525747035, -5.940338138344011, -5.40158748887255, -6.8282736444630405, -6.302506440698851, -6.49077786629904], [-5.042027972011844, -4.299718672656642, -4.243324982365076, -1.9825142998026875, -4.725280240738409, -4.751810055033314, -5.528415080508104, -5.372269658882421, -5.58057814379269, -5.634199235231788, -2.092585085927561, -5.359310514239916, -4.924244936355461, -6.182928172882705, -4.358300013368391, -5.644135441890917, -5.3193472574280785, -2.4216711986248063, -3.5454438576615357, -6.6553797837670805, -6.406683158028254, -3.42063060974259, -6.230496589801815, -5.313180742512414, -2.1141313157147192, -6.38251279710044, -7.199431055161392, -3.210947634389801, -4.823896962521179, -5.599497627038855, -3.7500872171087103, -6.903338504201014, -6.960201366095558, -7.542682978767983, -5.613234106766742, -6.36476085164199, -3.6714819678600925, -4.875454772186028, -4.20471280907303, -3.884017303250544, -3.4828969984685907, -4.20471280907303, -5.449056802096191, -3.411103801364231, -4.476181634325785, -6.299489461288767, -4.386290645201218, -5.042027972011844, -4.789932598747867, -5.333885415874105, -4.1803254304220925, -4.950988261330972, -6.251222720318932, -4.73156059252596, -4.355150404465494, -4.750641830601388, -5.174666905853347, -5.78646844695934, -5.852762894799176, -6.180486169227153, -6.353098911894147, -6.096324376905443], [-3.621153540136533, -3.927545190670303, -4.761298021301781, -5.2327070159671, -5.390836480884121, -5.489330016767771, -5.419938023343771, -5.638503085521277, -4.333907884333748, -4.5925613698517225, -2.3774699118392917, -7.240027999198052, -5.856199730573355, -5.309006462636489, -2.1188535037876086, -6.988713570917146, -7.259929153515347, -5.733217613565709, -2.1988937611323722, -6.914879308906094, -7.285375819176511, -7.471829613255376, -6.297483936486472, -6.369085087302357, -3.0144535413397078, -6.131611710433321, -8.510182710405289, -3.290870578196505, -5.79050447130796, -7.191941812530414, -3.661034886200146, -5.933160771709483, -6.992552347224311, -6.430741168725453, -6.006448567479316, -7.6405790925037875, -3.820123280601132, -4.60874181430768, -4.552214016916188, -4.152125265844404, -1.8440074384079603, -4.33229280995845, -4.748982594711727, -5.590043992369601, -4.083551769494654, -5.885005729822436, -5.570371226770897, -4.35431953325285, -5.037889407059404, -5.600505333614971, -5.2234842891983275, -5.239347146606377, -6.012203979049936, -3.2506791222758458, -3.493367719618491, -4.386449689183688, -4.5929103490909995, -3.6635121789617022, -5.259536257638402, -6.071656848963408, -5.913358144413303, -6.637031915632404], [-2.697794466436189, -3.604662393389068, -4.011263957500705, -4.728080882146555, -4.781372179196779, -5.33155887684335, -5.439355447299057, -5.586922158039964, -5.370633615250003, -5.513150772897161, -4.398260316869362, -5.11207163673225, -4.190082883743458, -3.4501177344839546, -6.597640067453566, -5.490434815062624, -5.150911470048515, -4.837107457567751, -2.85030853825846, -7.320346050255055, -5.997701716211621, -3.4753179136511876, -4.2866759373606715, -3.397053452568678, -6.703360230660674, -4.113003141200578, -6.444877312901155, -3.074043686998346, -3.452623303723489, -5.193702907090925, -3.9327969395172757, -4.131242383720524, -5.418238523858134, -3.7478131640115406, -6.184652086682414, -7.063123185287982, -3.7394230709797665, -3.4179694218787144, -4.267288710081795, -3.0709940324857143, -4.263518677341253, -4.701005637323521, -4.417094629495229, -4.569236359692398, -4.473887479448818, -5.8432973258667005, -4.728080882146555, -3.754953367214181, -4.1263095154005, -3.415012032977706, -4.81618158386264, -4.691820172692138, -5.666572324000903, -2.795735372125381, -3.2437129699651224, -3.9019638438387987, -3.582993942690341, -5.112804505947904, -4.894364079117085, -4.722110715160052, -6.056219323109372, -5.5022692727096265], [-4.619141531251806, -4.242478414047348, -4.655112011634125, -5.260049492712331, -5.018630839221239, -5.434973330528597, -5.291682300504695, -5.652426966065128, -4.133100006910822, -4.834429582625707, -1.998550183209591, -5.7433143168994505, -6.845817661060527, -6.562531200081784, -2.4549848852751466, -6.626189051853761, -6.290387731057782, -5.567486289416045, -2.141146196531367, -7.242549089199937, -7.104679294976816, -2.8517566389677937, -5.192894636686687, -7.082206439124757, -2.3092018462234853, -6.71448165899944, -8.017189304011065, -2.1705324264442973, -5.663122255181876, -6.392769837961348, -2.584044215625091, -7.28700085177077, -6.816997222525035, -7.454881724409931, -7.498844847831047, -6.3210545294507545, -3.9234738704665255, -5.210974225191004, -4.156261002958894, -4.841496749848799, -4.597299789336757, -4.421170202790724, -4.975365923256807, -5.672123216040852, -4.474317672329112, -6.090327349926446, -5.367408011032831, -4.491383872387231, -5.280618880960439, -5.815780620152271, -4.640826054386648, -5.541761398177608, -6.307308608546119, -4.360911011272527, -3.968855752517438, -5.060904184340462, -5.092811178129705, -5.9935197902566175, -5.119506808592894, -5.086750553518014, -6.052587021943599, -5.38489332537385], [-5.261046281621696, -4.9148297535258045, -5.093521542269181, -5.160962823064713, -5.492794680129424, -5.583178741597693, -5.608843109973599, -5.828958322395465, -5.818217080564052, -5.6351835265880075, -1.9431063436877503, -5.762127613913009, -6.364566555601998, -6.310693565661846, -1.465488331318766, -6.653479397439911, -6.983078754773362, -4.701955198551832, -2.920630198813372, -7.381717897811127, -7.114655112562081, -3.628618850962897, -4.741931970713727, -6.050483313874263, -2.961525855445553, -6.683481647743711, -7.823550650090166, -2.1208799042866553, -6.236585593508124, -6.735576759627112, -3.3232120191551315, -6.891511561247872, -6.08336298799342, -7.9413336857465495, -5.554867108771802, -7.122498290023107, -4.193090909121505, -3.901105949199606, -4.152689701478908, -4.935827900365578, -3.224590180736388, -5.347946392964479, -5.205112607677643, -4.550306897288771, -4.485129674572183, -6.303724896345753, -5.654950567328784, -4.928779413034516, -5.0770940681272325, -5.30867433061095, -4.570963767879653, -4.7737511552658995, -6.146602446129337, -3.6472116840395126, -4.583317229489316, -4.247999881283233, -5.050961927850385, -5.554867108771802, -5.239253651630843, -5.752077278059507, -6.0889495947020595, -6.114482896707225], [-4.8357984388643835, -4.657872493543467, -5.438431611883497, -5.481947249319659, -5.480513557317811, -5.739017089171186, -5.670890393572083, -5.859121927588727, -6.047318448825703, -5.930658377913943, -1.7234534048021994, -6.808376835001858, -7.018097365983928, -6.515303913915201, -2.0800327647225068, -6.985307543160936, -6.434021280478344, -5.690138581877, -3.0546210789668655, -7.433612809945593, -7.609892052283585, -6.846949109788098, -7.145930737493813, -6.771237288052402, -1.3242305561151027, -6.88123818326673, -7.934388097858083, -6.599387031125742, -6.8029859863669815, -6.569147145936024, -2.3927324727103167, -7.200418922777882, -7.33738477785104, -7.62201341281593, -4.893045313052039, -5.366877919534872, -4.1899951004809015, -5.453656819480563, -4.694403309779647, -3.431804500645092, -4.0124147615767685, -5.525942614164559, -5.309719505694924, -5.568828205702649, -4.124028746206436, -6.415604553692113, -5.4275025413513065, -4.984699762805498, -5.237511197353998, -5.157641365469638, -4.101047614464405, -5.085610237260755, -5.550223017871613, -4.869440755282619, -4.489173830779152, -4.234321454353582, -5.041242413079193, -5.780689785571754, -5.231908941805328, -5.964947451392574, -5.752089170738539, -6.139854701747302], [-5.508465007700513, -4.839258034686576, -5.109472414097966, -5.519275923804729, -5.707703038161169, -5.439902336263216, -5.7235763873174585, -5.821556795677663, -5.734300350680434, -5.610247702010455, -5.910709501108635, -5.7235763873174585, -4.435262434557772, -3.8610478472011964, -5.758858201462099, -6.0632537548876195, -6.044700346991872, -4.117878794499082, -6.575328598054043, -7.243783166023616, -6.403478341127383, -5.083829983484629, -3.317712944869911, -1.7163886672099804, -5.622223893057171, -5.543479525545615, -7.373836294271814, -4.235327303674574, -3.4757995308184078, -3.8216678856997803, -6.863010670505823, -5.619817153026606, -5.966922645949188, -7.096625521687328, -7.1845942946332855, -6.5629060780554855, -3.9611313057581787, -4.164449795424729, -3.6745940531223313, -2.809054169189011, -3.303860323066671, -4.356804692686432, -4.075943929085389, -4.993775889109478, -3.5055329961999946, -5.4956443192714515, -3.896020865817486, -3.604273312212492, -3.5622476861654384, -2.789152302491872, -3.9102728885246876, -3.0660244750660355, -5.631909198791635, -4.404134333185408, -2.9033092297609358, -3.155554830818952, -5.387104150696246, -4.341290047595106, -5.258585299895267, -5.109472414097966, -5.821556795677663, -5.14024407276472], [-4.364789554126129, -4.918384854279969, -4.918384854279969, -5.1613310328903585, -4.587530609962979, -4.784853461655446, -5.006160465162354, -5.0403784224726955, -4.994002260682545, -5.402493089707247, -1.4796200682550344, -6.181000119218348, -6.701776073837507, -6.252825853789604, -2.1508860947624866, -6.086590434747274, -5.772240115213331, -5.482914657017821, -3.9652464014134288, -6.426672783592586, -6.401671481387169, -6.285261129542758, -5.890845857621178, -6.3772800282630095, -1.9660163181458141, -6.770768945324459, -7.107241181945672, -5.362398632505369, -5.920923312858457, -6.7530693682250575, -2.0597919147266994, -6.701776073837507, -6.505661194911217, -6.5476253940102485, -6.637237552699936, -6.7530693682250575, -3.9354569646113213, -4.092932522818745, -4.814706424805127, -4.961309898997003, -4.355174095426687, -4.949681861001884, -4.869194610089197, -5.046727650151355, -4.8507000274530325, -5.115948027042541, -4.814706424805127, -4.689097748317353, -4.863875448611597, -5.043547997233975, -3.6732539774605253, -3.9481153614832447, -5.340799520701906, -4.835115296436334, -4.068688911208753, -5.024679512929592, -4.753487802377227, -4.994002260682545, -5.247044551164388, -4.698046353893368, -5.307182909902921, -5.227776132298511], [-5.115394338583977, -4.7561420736309445, -5.007842676853897, -5.66123448019507, -5.033665707964376, -5.645764558422938, -5.535809351386801, -5.907471144471458, -5.828086115216893, -6.105296887801378, -1.6832402388074508, -7.437524027650993, -5.590313253886724, -6.862982589498895, -3.0491034044747836, -5.586172461220692, -6.632458930887062, -5.218606625703514, -2.4710973040085813, -7.398809515470303, -7.268447697607059, -2.7116510959031834, -6.686853002952861, -3.7188303108266303, -1.9217106058640063, -6.098376444956805, -7.548749662761217, -2.711068175455316, -6.088084758920257, -7.1630871819492326, -2.4953178542046213, -6.5371487510827375, -6.531815405107375, -6.592107635363495, -5.218606625703514, -7.464192274733154, -4.025855346121542, -5.098759278079445, -4.812664759541027, -5.369662433402117, -3.845996309584077, -5.031290409935469, -4.627804324836165, -5.549657248245595, -3.554062953084761, -6.151486270270753, -4.8426621708268565, -4.640563668589926, -5.069989313821911, -5.128381534110789, -4.6191257693799175, -5.164313543336852, -5.9721996171719525, -5.146851716173189, -4.482065653234942, -4.222339216728002, -5.190396010216277, -5.978297197040071, -4.681519433426748, -6.057830350562455, -5.619789072019677, -5.749664153441089], [-4.867406343870729, -4.4533740975545415, -4.953434176237852, -5.384635998990067, -5.328952912513263, -5.6593669190947695, -5.597013688377717, -5.757806991908022, -5.710273291442284, -6.117674508577314, -2.0752354231922467, -5.330274789771179, -7.239817294655618, -4.653127950557675, -1.9861891376978962, -6.498730651806554, -6.748830013109291, -5.799220777458779, -1.8880861157511444, -7.060476365999801, -6.592340150928861, -6.615981913985901, -5.98705432616025, -6.055682832549346, -2.2283490142554703, -6.244583360441343, -7.060476365999801, -6.9543698600052535, -6.005072831662928, -6.675112437212724, -3.324941220245065, -6.737960340872387, -6.846328377594169, -7.276184938826493, -5.466076330930241, -7.601607339261121, -3.2284338811241238, -4.7571452826690335, -4.2772171644143535, -3.794375211195705, -2.9321390456478555, -4.655146114713912, -4.556474587206883, -4.658518799192551, -3.9133691163329902, -6.159223511490186, -4.874928318325417, -3.9851607325065794, -4.593769151695071, -4.865742449976033, -4.192747084853396, -4.158562847663465, -5.583291346122616, -4.6687056444995445, -4.403457644910037, -4.135470875670939, -4.338591952168042, -5.484425469598437, -5.10713123845697, -4.666659956776251, -5.507845743806536, -5.56639905255811], [-5.367738120634485, -4.778432275797196, -5.153988612512787, -4.9880684755260525, -5.022992659762126, -5.315533041364984, -5.937669186281741, -6.10637419308453, -5.985149724514735, -6.0666028696576335, -1.4917500304139726, -5.449683054485573, -5.538947192058466, -5.596026010244295, -2.1356770840200325, -7.153812526227932, -7.094389105757131, -6.013018623523528, -2.2775252846627545, -6.010847070010019, -7.246405313055757, -6.755173383190167, -6.697507741340358, -5.530860894627108, -1.9981251565724831, -5.698525293621089, -8.083802102460249, -4.860110306811463, -5.882753428685626, -6.945748081740843, -2.8863534391722627, -7.100819996087421, -6.962461562714583, -7.231590227270615, -4.4224533361891325, -6.0688990819179836, -3.292438554454217, -4.782234561746934, -4.459461169483883, -4.359771877270199, -3.9163352754091854, -5.15214868559078, -5.0863471755948115, -5.622152314836516, -4.041658687615878, -6.321199217523649, -5.22160122153078, -5.082053480720112, -4.742403534262838, -4.896452531238822, -4.577416633798337, -4.325814840936012, -5.22258692885554, -5.570564946046022, -3.7753199382595333, -4.751597592285044, -5.351900685535859, -5.538947192058466, -3.6838339356894148, -5.717756655548977, -5.9396873504379775, -6.193602560418941], [-4.999514188504179, -4.700587505853424, -4.67022803135639, -4.81576740495976, -4.92872993147181, -5.418517964945142, -5.4525477135314535, -5.471965799388554, -5.601583292691941, -5.294558257397029, -2.0675057720915833, -6.606510097282728, -6.66295140818768, -6.268297216183731, -1.8472066532219233, -6.801101746668497, -6.464629510047284, -5.518112402024337, -2.6153829107633095, -5.40926720517299, -7.042937430994859, -6.122880216325227, -4.984950824316282, -6.978988706394586, -2.5341534752438712, -6.335738496979263, -7.22903971062872, -5.047082605423289, -6.606510097282728, -7.195517018590076, -3.3881221963080814, -6.764734102497622, -6.935691900641261, -6.06802185577318, -5.518112402024337, -6.21743879895024, -3.306166689176508, -4.5835098665078435, -4.096018906794089, -2.7778687007936327, -3.149770331333854, -4.522505810044474, -3.6065186143693406, -4.579485716208118, -3.9009018998360694, -5.695109350702764, -4.592420415540891, -4.39850374434322, -4.858371250530563, -4.687974726037726, -4.284600731462279, -4.828050056815601, -5.7714823294873385, -4.760362534007699, -3.6910492160852075, -3.1337486268025883, -5.114702793098268, -5.044521784561615, -4.990993138772245, -5.21262268525524, -5.1495981689488834, -4.939033399841534], [-5.573013579156251, -4.7557456323553735, -4.835888625843566, -5.4310032481645445, -5.375816832597254, -5.748275644237248, -5.617589203744957, -5.426286258286405, -5.527732874623095, -6.027336973645122, -5.771265162461947, -3.4363190171520683, -4.863649827007062, -5.342810536129083, -3.9603112459777443, -3.1741279621323484, -6.522548369609561, -4.089429329592385, -5.667234434234122, -6.573100648772392, -5.74180312973163, -3.6814950844272993, -4.555731535734934, -2.60141208694855, -6.057840427938536, -2.6538446056843625, -7.3352407008192895, -2.4633202784204524, -3.0508881488790776, -5.069846636151324, -3.6495408996227328, -4.548858656447172, -5.7613477258046, -6.031638055544512, -6.61856302284915, -5.764642621701453, -4.906092884516815, -3.945751808422034, -4.075923906392578, -4.145352412824341, -4.3284585910787134, -4.1375398730875474, -4.386633312115669, -4.864991208831262, -5.2459274106602996, -5.389330551763976, -4.366047982499911, -3.5236520927129136, -3.6216686341149815, -3.0467401241562313, -4.4457511661828155, -3.28056139498962, -5.728982441302569, -3.2282776746379773, -3.3483163872180257, -3.6120365041642786, -4.280940073114231, -4.240597235287379, -4.439591885822281, -4.571620648562269, -5.13801612348307, -5.583972592945972], [-5.648871675811755, -4.741030782968909, -5.070478143556586, -5.038332518583169, -4.984450311189462, -5.652113169735926, -5.878916005474925, -6.018576120040821, -5.698633185370818, -5.924757364823166, -1.8468585970227798, -6.844251516414859, -7.126600242891277, -6.081398705962876, -2.934905888757429, -3.6595354362414967, -6.817864761241664, -3.5921374614243615, -2.9623018840998165, -7.15537920744132, -5.837035508229937, -2.956367800752568, -5.44072677540776, -6.135188880549014, -2.35874447462609, -6.817864761241664, -7.744626792983114, -1.829808862763535, -4.656579250518651, -7.012765100242478, -3.346772137573546, -6.156466278996299, -6.772055225210369, -5.459294948136628, -6.2464145156592386, -6.772055225210369, -3.9054574859103, -5.259720143195114, -3.484916480113615, -4.59888775210554, -4.26703708287839, -5.280773552392946, -4.568219053541838, -5.236954932422101, -4.227597595795837, -6.429913235626207, -5.14878310004672, -4.299244656170703, -5.066854951187166, -5.722730736949879, -4.343868104023761, -4.895290258966649, -5.924757364823166, -4.2366225736701635, -3.439140235431567, -5.290903070631802, -4.913738130330258, -5.32777360644013, -5.700335313441349, -4.118883335232663, -6.285399962372192, -5.825384891009962], [-4.134798672989471, -4.089847285127205, -4.1376763728170864, -4.175860159787245, -3.5867437074551245, -4.073498147125676, -4.247208240584712, -4.485742942219219, -4.161001045383496, -4.76664532768562, -4.782994465687151, -5.224827217966189, -5.79734641073752, -5.371880635922686, -5.362028339479674, -4.805217602471861, -5.696541711615555, -5.1428140663053545, -4.243997964954463, -5.696541711615555, -6.074978147335799, -4.388579193765571, -4.887134724939747, -5.433124261163405, -5.127188748402274, -5.465212575714905, -5.5681605449673475, -3.871109027280912, -4.3522115495946965, -4.887134724939747, -1.4812734394569864, -4.9697213159490214, -5.42265296129611, -5.5681605449673475, -5.696541711615555, -5.592551998091507, -3.9025017396887742, -4.303421385425264, -4.377529357578986, -3.2036759521599856, -4.414847120586181, -4.370230055097374, -4.283218678107745, -4.293269013961246, -4.570900750559526, -4.639893622046477, -4.473572406598963, -3.944368319081564, -3.9659778034144195, -4.663991173625537, -3.393956618621509, -4.489832927470744, -3.9232159440763374, -4.221810049979101, -3.677082874537429, -4.095356940938174, -3.0285527594662907, -4.570900750559526, -4.348646483430199, -4.449666885745409, -4.481669616831582, -4.21556002963393], [-5.468543074118657, -4.3409301123745765, -4.939132602788878, -5.030955774234888, -5.486636817561356, -5.444018684255573, -5.476853371252285, -5.808607963491769, -5.908385471304435, -5.666153323427702, -2.5849374499856705, -5.739356727450997, -6.452595908093631, -6.391754248837386, -1.3630653142736813, -7.218432822021854, -6.741508749931544, -4.321907198551311, -2.3295168721177877, -7.554905058643067, -7.600367432719824, -7.660265574300893, -6.068262723688894, -7.064282142194595, -2.4089621725210324, -6.261701120589188, -7.218432822021854, -7.400754378815808, -6.405740490812126, -6.66108718262097, -3.202908170420881, -7.345184527660997, -7.318516280578836, -7.17951740577218, -6.2346317986209705, -7.685266876506311, -3.523815890500982, -5.001129001779655, -4.254551658580372, -3.1791480369827805, -3.272238460048792, -4.979751531007288, -4.696520976240299, -4.996816791561475, -3.7821441205484283, -6.068262723688894, -4.767838317222632, -4.961310103104565, -3.3169412686460493, -4.254142074302682, -3.686717843919221, -4.781622132973809, -6.395232513213711, -4.882290151848487, -4.035101934573064, -4.142657840794326, -4.886134831709296, -5.154991621688459, -5.261248596393322, -5.763145589415012, -5.307128202144016, -5.4187681732866855], [-3.7153348719791315, -4.23835637018879, -5.401341822408315, -5.655381048015406, -5.3230099791384085, -5.723838105173554, -5.787159933132683, -6.191069830974041, -6.118044695959151, -6.512733837722823, -2.85919837658876, -7.902248813542757, -2.4061387894065027, -6.693770047296114, -2.250325724429897, -7.350962550860164, -6.005451461342365, -3.2057646919026945, -3.1269276063809657, -8.282234836349936, -4.929356031958588, -4.893377430923357, -4.350779231381914, -5.039126698093967, -3.0950598675329433, -2.702149075140166, -7.065140944061303, -6.4656293685353505, -6.6671525062961585, -1.5532677412410396, -3.407685858090993, -6.917395410231305, -4.818860209735889, -8.330553413620743, -4.697144805661785, -7.44034239452964, -4.201345166462018, -5.036551747250591, -4.447052124631202, -4.271306064673767, -4.3875307527168115, -5.68024420720805, -5.463555006909888, -4.6680077696105, -4.729585831829687, -6.877248342217893, -5.7894972929813875, -4.744562286019245, -4.863271676535759, -5.996776708148691, -5.115645847928164, -4.782587554878788, -6.69569868820252, -5.6198664278662305, -4.372289459396023, -3.8304137647360905, -5.143464614214886, -5.692911512408815, -5.442384299786921, -6.144390895990383, -6.553756817069401, -6.771887826560066], [-5.439192275092616, -4.781176186054329, -4.766861598657258, -5.578787193555129, -5.543563426336324, -5.577507602849504, -5.742587353208953, -6.240425781448133, -6.11031213333427, -6.461710020172159, -2.6665339561656087, -5.7471225083743445, -6.597455272456702, -6.655866034613116, -2.1677366785812078, -7.063212610820987, -6.615344837207477, -1.8542823223629723, -2.8203019407701473, -7.921874229858505, -5.964600322152877, -7.4190807777897785, -7.115398363991557, -7.50316389500032, -2.434370037607787, -6.75042461723813, -8.501692725111447, -2.436966652419961, -5.508344242912733, -7.262628600974241, -3.3318853658660776, -6.320468489121669, -4.662752576421778, -7.96269622437876, -4.3951216252530045, -7.773454224740232, -3.4248833433841024, -4.6171877485771935, -4.856615893655904, -5.148452521314832, -3.262215858586344, -5.067705520626301, -4.995134827791466, -4.808655260848847, -4.056803079326151, -6.372894286461519, -5.283769734998518, -4.869131701587735, -4.356292992089062, -5.580068423711178, -4.205055407055328, -5.300108397353307, -6.413362236014366, -4.081705328198343, -3.5651654031689133, -4.4607321960690065, -4.914213385439241, -3.3562768189902368, -5.301077859377855, -5.2191716350828905, -5.66568217643417, -5.148452521314832], [-4.692472771166889, -4.3008934499719516, -4.549181741142988, -4.955265087292482, -5.083773336506403, -5.36724487642184, -5.286555965171697, -5.606625988867734, -5.3561026998685985, -5.6766361544403825, -6.310506725070788, -5.345083304618988, -6.354631529979725, -5.436838168221035, -2.778870257014558, -5.888394383529467, -5.518132910614282, -3.971194340356585, -5.541198183545278, -7.353160360090853, -5.418980550821029, -3.590537431731367, -3.3241709789128793, -1.8944436557320132, -5.91226186493611, -3.390599447549114, -7.2559966116372046, -3.6807421830309885, -3.742794171598555, -5.40143624117012, -6.914247317915148, -6.268246915780905, -5.966865998970962, -6.802329401711162, -7.18453764765506, -6.063197107909394, -4.040088147996806, -3.4233375319614976, -2.7095546985241823, -4.374927246403353, -3.562959147374605, -4.404434128912793, -4.579296940419071, -4.391681565015401, -4.438151297809303, -5.606625988867734, -4.3282656378478706, -4.1449534915722195, -4.0950638220693705, -3.3913758451036427, -5.0164696546102965, -4.409796072054178, -6.030044900592494, -3.187603169643488, -2.500811269450856, -3.4121731416766146, -5.192891489277063, -4.804034117568558, -4.273312053475689, -5.056716535118889, -5.514880875227905, -5.219114684376165], [-5.262101982971463, -4.751704716461968, -4.4750082149096455, -5.006514043913175, -5.07515161250067, -5.251452255054804, -5.285938431125974, -5.392274080942006, -5.414469813333791, -5.840086736879263, -2.606359904105076, -6.823463761969788, -6.89757173412351, -5.234645136738424, -1.8016665052081617, -5.931967688708849, -5.7146991018376605, -5.437169400849897, -1.672652532530341, -6.72630001351614, -6.773702252410724, -5.308112288620296, -6.00575385876808, -6.6810434219280195, -2.3521748840468923, -5.96169386897405, -7.419447194076086, -6.095225261598572, -5.008173795331539, -6.397795946544104, -5.514028406375509, -5.907173030095633, -6.070893160939042, -4.572171963363293, -6.91979487090822, -6.823463761969788, -3.666029218824578, -4.169934399745995, -4.315857524665681, -2.6071122929751187, -3.822642418908326, -5.326212330263914, -4.3600448319184775, -4.917707706137853, -3.8075287810982776, -5.953110125282659, -5.187841070540621, -4.792366055507543, -4.8403533292283845, -5.164264339976624, -4.320022461964966, -3.8952676954062846, -5.745470760504414, -4.036941945426083, -4.228361366337563, -4.708930900478682, -5.247223918945283, -5.006514043913175, -3.8952676954062846, -4.574322501826521, -5.886970322778113, -5.748949024880739], [-5.49714261021104, -4.668772759709356, -4.7793028170607235, -5.269011517385791, -5.403728165571614, -5.600190986110487, -5.607143505425369, -5.736034518493389, -5.749350294469161, -5.980569259788917, -1.8463761496649909, -6.7957313909118255, -6.842614976810675, -6.277301167760615, -1.6016725181239624, -6.834646807161499, -6.483542908272222, -3.3781298559126016, -1.9147238137262874, -7.264209466848723, -6.97044834832056, -6.2503937148406905, -6.323821183395508, -6.466922027036182, -2.304012035094697, -6.232849405189781, -7.182292344380837, -4.500040235600472, -6.194465162181466, -7.045955900828706, -3.7335540179496816, -6.633976111699347, -6.224191342446667, -7.560054849939665, -6.541029999189903, -7.7391030813886506, -3.9953827115751284, -5.225208894727398, -4.2623981472183505, -4.983820106246462, -3.5900722944719905, -4.880831478599072, -4.625152137233465, -5.350160419862394, -4.464551422640724, -6.03257404477923, -5.25911044640308, -4.290672925686517, -4.802954305142684, -5.287422079228971, -4.798800612773991, -4.757191363453292, -5.977196575310277, -4.844383490556383, -2.946883968159507, -4.887603513509017, -5.795997983260946, -5.6188395451885595, -4.719156549714065, -5.921535728204724, -6.1778672707724285, -5.694577804815043], [-4.469097220436953, -3.6430767584970343, -3.7844067498952314, -3.931680914923784, -4.272838155799217, -4.129621978555971, -4.297357772973536, -4.384369149963165, -4.338992745625113, -4.178787025578902, -4.976420213651742, -4.292052720743842, -5.461928029433443, -5.887479896108345, -4.57562801124297, -5.526466550571014, -6.262964172004573, -4.4462034005711, -5.238784478119233, -6.017291507630335, -6.355745905455539, -5.968972930359527, -5.520424236115051, -6.328346931267425, -5.70515833931439, -4.218953067304238, -6.6434279779073195, -5.648805402763258, -5.5635078222513625, -4.870360641691417, -6.412904319295488, -1.949196049230905, -5.968972930359527, -4.4990139814749455, -6.027241838483503, -6.442757282445169, -4.057336183871974, -4.448263133534111, -4.000410247075965, -3.7940068236242506, -4.150834895195163, -4.166267300233975, -4.566303934367847, -4.3901887590164295, -4.028468199871122, -4.376661982718227, -4.915384323065373, -3.014652447863089, -3.369082406994758, -4.61381880997536, -4.55706599338291, -3.5814225774035497, -4.577972678202224, -4.001728638829222, -3.270401473211861, -3.264703452097223, -3.9718368746319084, -3.132988733879366, -4.361423592613295, -2.595191993338583, -4.516519898988561, -4.912100247864183], [-3.9699474093476774, -4.189042372135887, -4.058989243887689, -4.102030995146257, -4.171520021443684, -4.318254103615893, -4.424253392850214, -4.758810531535327, -4.504029895277935, -4.414714369803455, -3.0879080140350674, -5.469976217597951, -3.790334046490602, -5.366435538657111, -3.235086137375257, -5.727805326900051, -5.408282648592611, -4.971959552124542, -5.070590155566169, -5.076706382583605, -6.181472536826093, -5.921961341341008, -4.9999725883522155, -5.5070174892783, -1.5568628976878147, -5.5070174892783, -6.110013572843948, -4.0678979887767985, -5.089052218405905, -5.200643283814367, -3.6661684098537215, -4.557328477002297, -5.479108701161223, -5.159821289294111, -5.479108701161223, -5.616579691789827, -3.7152580200502454, -4.23556238777078, -4.212031890360586, -3.866007780968386, -4.0678979887767985, -4.4633463186414915, -4.142178063159669, -4.318254103615893, -4.276053749125516, -4.772293881872614, -4.332726136224427, -4.11133338780857, -3.9089648990693253, -4.309670359924501, -3.9127889955077286, -4.338574106106851, -4.528549512452254, -4.149433234040841, -3.5292744136497762, -3.6009751629760713, -4.350373653038005, -4.828122331425556, -4.047963773875981, -4.665125047458004, -4.673288358097166, -4.633122316371831], [-4.806719860900339, -3.6507473420444714, -4.429814962058902, -4.347330952102998, -4.3949170222076255, -4.698366552912107, -4.809448375553543, -4.35423944244681, -4.534137594904308, -4.3611959922401695, -2.871768725634745, -4.598399529122678, -5.564587232141456, -5.519124858064699, -2.4250566837898457, -5.688201188108633, -5.967149580591659, -5.01834957015221, -2.505498094634157, -5.3939617151106924, -6.3813483686685775, -5.360223575478843, -5.821732580733155, -5.742268409378909, -2.5427453130265225, -6.0299504818306895, -5.949907774157153, -6.048642614842842, -6.106911522966818, -6.522426966928483, -1.9368412502165815, -5.384205540165328, -3.2469989725553616, -5.166904264475346, -4.662348253722951, -5.93295821584338, -4.052787789271272, -3.8964417188805776, -4.725629703670472, -3.994277013931218, -3.570836586604418, -4.883136091344702, -4.20182336843176, -4.894970548991704, -4.1342203512338624, -5.159122124033291, -4.442985425248647, -4.740819869164446, -4.583031498894364, -4.589588899440523, -4.4058064220068935, -4.87144005158151, -4.8541485544714496, -4.806719860900339, -4.241282205172307, -4.937530163410501, -4.382360803432213, -5.0116381355642226, -4.368201274828579, -4.782492565565015, -4.544565218066568, -4.4563316672015425]],
probHashThreshold: 0.015,
probHashChars: {'1': 1, '0': 0, '3': 3, '2': 2, '5': 5, '4': 4, '7': 7, '6': 6, '9': 9, '8': 8, 'A': 36, 'C': 38, 'B': 37, 'E': 40, 'D': 39, 'G': 42, 'F': 41, 'I': 44, 'H': 43, 'K': 46, 'J': 45, 'M': 48, 'L': 47, 'O': 50, 'N': 49, 'Q': 52, 'P': 51, 'S': 54, 'R': 53, 'U': 56, 'T': 55, 'W': 58, 'V': 57, 'Y': 60, 'X': 59, 'Z': 61, 'a': 10, 'c': 12, 'b': 11, 'e': 14, 'd': 13, 'g': 16, 'f': 15, 'i': 18, 'h': 17, 'k': 20, 'j': 19, 'm': 22, 'l': 21, 'o': 24, 'n': 23, 'q': 26, 'p': 25, 's': 28, 'r': 27, 'u': 30, 't': 29, 'w': 32, 'v': 31, 'y': 34, 'x': 33, 'z': 35},
userTransitions: {
search: {}
},
searchEngines: [], //Variable for content extraction fw.
rArray: [], //Variable for content extraction fw.
extractRules: {}, //Variable for content extraction fw.
payloads: {}, //Variable for content extraction fw.
anonSearchEngines: [], //Variable for content extraction fw.
anonRArray: [], //Variable for content extraction fw.
anonExtractRules: {}, //Variable for content extraction fw.
anonPayloads: {}, //Variable for content extraction fw.
messageTemplate: {},
anonIdMappings: {},
patternsURL: 'https://cdn.cliqz.com/human-web/patterns',
anonPatternsURL: 'https://cdn.cliqz.com/human-web/patterns-anon',
configURL: 'https://safe-browsing-proxy-network.cliqz.com/config',
searchCache: {},
ts : "",
mRefresh : {},
can_url_match :{},
ismRefresh : false,
activityDistributor : Components.classes["@mozilla.org/network/http-activity-distributor;1"]
.getService(Components.interfaces.nsIHttpActivityDistributor),
userTransitionsSearchSession: 5*60,
key: ["source","protocol","authority","userInfo","user","password","host","port","relative","path","directory","file","query","anchor"],
q: {
name: "queryKey",
parser: /(?:^|&)([^&=]*)=?([^&]*)/g
},
parser: {
strict: /^(?:([^:\/?#]+):)?(?:\/\/((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?))?((((?:[^?#\/]*\/)*)([^?#]*))(?:\?([^#]*))?(?:#(.*))?)/,
loose: /^(?:(?![^:@]+:[^:@\/]*@)([^:\/?#.]+):)?(?:\/\/)?((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?)(((\/(?:[^?#](?![^?#\/]*\.[^?#\/.]+(?:[?#]|$)))*\/?)?([^?#\/]*))(?:\?([^#]*))?(?:#(.*))?)/
},
activeUsage : 0,
activeUsageThreshold : 2,
pageStatusCountSent: null,
actionStats: null,
actionStatsLastSent: null,
bloomFilter: null,
bf:null,
strictQueries:[],
oc: null,
SAFE_QUORUM_ENDPOINT: "https://safe-browsing-quorum.cliqz.com/",
SAFE_QUORUM_PROVIDER: "https://safe-browsing-quorum.cliqz.com/config",
quorumBloomFilters: {},
_md5: function(str) {
return md5(str);
},
maskURL: function(url){
var url_parts = null;
var masked_url = null;
// Fix
// url_parts = CliqzHumanWeb.parseUri(url);
url_parts = CliqzHumanWeb.parseURL(url);
// TO BE FIXED
if (!url_parts) return '';
// Fix
if (CliqzHumanWeb.dropLongURL(url)) {
//Explicit check for google search url.
if(url_parts['hostname'].indexOf('google') > 0){
if(url_parts['query_string']){
var query_stringsParts = CliqzHumanWeb.parseQueryString(url_parts['query_string']);
if(query_stringsParts['url']){
masked_url = query_stringsParts['url'][0];
masked_url = CliqzHumanWeb.maskURL(decodeURIComponent(''+masked_url));
return masked_url;
}
}
}
masked_url = url_parts.protocol + "://" + url_parts.hostname + "/ (PROTECTED)" ;
return masked_url;
}
return url;
},
isShortenerURL: function(url) {
try {
var url_parts = CliqzHumanWeb.parseURL(url);
if (!url_parts) return true;
if ((url_parts.hostname.length < 8) && (url_parts.path.length > 4)) {
var v = url_parts.path.split('/')
for(var i=0;i<v.length;i++) if (CliqzHumanWeb.isHash(v[i])) return true;
}
return false;
} catch(ee) {
return true;
}
},
getTime:function() {
try { var ts = CliqzUtils.getPref('config_ts', null)} catch(ee){};
if(!ts){
var d = null;
var m = null;
var y = null;
var h = null;
var hr = null;
var _ts = null;
d = (new Date().getDate() < 10 ? "0" : "" ) + new Date().getDate();
m = (new Date().getMonth() < 9 ? "0" : "" ) + parseInt((new Date().getMonth()) + 1);
h = (new Date().getUTCHours() < 10 ? "0" : "" ) + new Date().getUTCHours();
y = new Date().getFullYear();
_ts = y + "" + m + "" + d + "" + h;
}
else{
h = (new Date().getUTCHours() < 10 ? "0" : "" ) + new Date().getUTCHours();
_ts = ts + "" + h;
}
return _ts;
},
isSuspiciousURL: function(aURI) {
try {
var url_parts = {};
url_parts = CliqzHumanWeb.parseURL(aURI);
if (!url_parts) return true;
_log(JSON.stringify(url_parts));
if (aURI.indexOf('about:') == 0) return true;
if (/^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/.test(url_parts.hostname)) {
return true;
}
if (url_parts.username!="" || url_parts.password!="" ) {
return true;
}
if (url_parts.port != "" & (url_parts.port!="80" && url_parts.port!="443")) {
return true;
}
if ( url_parts.protocol != "http" & url_parts.protocol!="https" & url_parts.protocol != "") {
return true;
}
var pos_hash_char = aURI.indexOf('#');
if (pos_hash_char > -1) {
if (CliqzHumanWeb.checkSearchURL(aURI) == -1 && (aURI.length - pos_hash_char) >= 10) {
_log("Dropped because of # in url: " + decodeURIComponent(aURI));
return true;
}
}
if ( url_parts.hostname.indexOf('localhost') > -1){
return true;
}
_log("Sanitize: URL is ok: " + aURI);
return false;
} catch(ee) {
// if there were any exception, we return true for safety
_log("Exception in isSuspiciousURL: " + ee);
return true;
}
},
calculateStrictness: function(url, page_doc) {
var strict_value = true;
if (page_doc && page_doc['x'] && page_doc['x']['canonical_url']) {
// there is canonical,
var can_url_parts = CliqzHumanWeb.parseURL(page_doc['x']['canonical_url']);
var url_parts = CliqzHumanWeb.parseURL(url);
if (!url_parts || !can_url_parts) return true;
if (url_parts.hostname!=null && url_parts.hostname!='' && url_parts.hostname==can_url_parts.hostname) {
// both canonical and url have a hostname and is the same,
if ((page_doc['x']['canonical_url'] != url) && (page_doc['x']['canonical_url'].length < url.length)) {
// the page has a canonical of same domain, which usually is a sign that is public,
// and the canonical is not the same url, which comes out of automatic generation
// of canonicals
strict_value = false;
}
}
}
return strict_value;
},
dropLongURL: function(url, options) {
try {
if (options==null) options = {'strict': false};
if (CliqzHumanWeb.checkForEmail(url)) return true;
var url_parts = CliqzHumanWeb.parseURL(url);
if (!url_parts) return true;
if (options.strict == true) {
if (url_parts.query_string && url_parts.query_string.length > CliqzHumanWeb.qs_len*0.75) return true;
// check the number of parameters too,
if (url_parts.query_string) {
var v = url_parts.query_string.split(/[&;]/);
if (v.length > 1) {
// that means that there is a least one &; hence 2 params
return true;
}
for (var i=0; i<v.length; i++) {
if (v[i].length>3 && CliqzHumanWeb.isHash(v[i])) return true;
}
if (CliqzHumanWeb.checkForLongNumber(url_parts.query_string, 8)!=null) return true;
}
if (CliqzHumanWeb.checkForLongNumber(url_parts.path, 8)!=null) return true;
}
else {
if (url_parts.query_string && url_parts.query_string.length > CliqzHumanWeb.qs_len) return true;
if (url_parts.query_string) {
var v = url_parts.query_string.split(/[&;]/);
if (v.length > 4) {
// that means that there is a least one &; hence 5 params
return true;
}
if (CliqzHumanWeb.checkForLongNumber(url_parts.query_string, 12)!=null) return true;
}
if (CliqzHumanWeb.checkForLongNumber(url_parts.path, 12)!=null) return true;
}
var vpath = url_parts.path.split(/[\/\._ \-:\+;]/);
for (var i=0; i<vpath.length; i++) {
if (vpath[i].length > CliqzHumanWeb.rel_part_len) {
return true;
}
if (options.strict == true) {
// if strict, check the no token in path looks like a hash
if (vpath[i].length>5 && CliqzHumanWeb.isHash(vpath[i])) return true;
}
else {
if (vpath[i].length>12 && CliqzHumanWeb.isHash(vpath[i])) return true;
}
}
var vpath = url_parts.path.split('/');
for (var i=0; i<vpath.length; i++) {
var cstr = vpath[i].replace(/[^A-Za-z0-9]/g,'');
var mult = 1.0;
if (options.strict == true) mult = 0.5;
if (cstr.length > CliqzHumanWeb.rel_segment_len * mult) {
if (CliqzHumanWeb.isHash(cstr)) return true;
}
}
var v = [/\/admin([\/\?#=]|$)/i, /\/wp-admin([\/\?#=]|$)/i,
/\/edit([\/\?#=]|$)/i, /[&\?#\/]share([\/\?#=]|$)/i,
/[&\?#\/;]sharing([\/\?#=]|$)/i, /[&\?#\/;]logout([\/\?#=]|$)/i,
/WebLogic/i, /[&\?#\/;]token([\/\?#=_;]|$)/i,
/[&\?#\/;]trk([\/\?#=_]|$)/i, /[&\?#\/=;](http|https)(:\/|\%3A\%2F)/];
// url_rel contains path and query_string
//
var url_rel = (url_parts.path || '/') + '?' + (url_parts.query_string || '');
for (var i=0;i<v.length;i++) if (v[i].test(url_rel)) return true;
// checks specific to the query string
//
// real query string (?) or a 'fake' one with a sharp on the path, they should be treated the same way
//
var path_query_string = null;
var ind_pos = url_parts.path.indexOf('#');
if (ind_pos!=-1) path_query_string = url_parts.path.slice(ind_pos, url_parts.path.length);
if ((url_parts.query_string && url_parts.query_string.length > 0) || (path_query_string && path_query_string.length > 0)) {
var v = [/[&\?#_\-;]user/i, /[&\?#_\-;]token/i, /[&\?#_\-;]auth/i, /[&\?#_\-;]uid/i, /[&\?#_\-;]email/i,
/[&\?#_\-;]usr/i, /[&\?#_\-;]pin/i, /[&\?#_\-;]pwd/i, /[&\?#_\-;]password/i, /[&\?#;]u[=#]/i, /[&\?#;]url[=#]/i,
/[&\?#_\-;]http/i, /[&\?#_\-;]ref[=#]/i, /[&\?#_\-;]red[=#]/i, /[&\?#_\-;]trk/i, /[&\?#_\-;]track/i,
/[&\?#_\-;]shar/i, /[&\?#_\-;]login/i, /[&\?#_\-;]logout/i, /[&\?#_\-;]session/i,
];
if (url_parts.query_string && url_parts.query_string.length > 0) {
for (var i=0; i<v.length; i++) if (v[i].test('?' + url_parts.query_string)) return true;
}
if (path_query_string && path_query_string.length > 0) {
for (var i=0; i<v.length; i++) if (v[i].test(path_query_string)) return true;
}
}
return false;
} catch(ee) {
// if there were any exception, we return true for safety
_log("Exception in dropLongURL: " + ee);
return true;
}
},
cleanDocCache: function() {
for(var key in CliqzHumanWeb.docCache) {
if ((CliqzHumanWeb.counter - CliqzHumanWeb.docCache[key]['time']) > 3600*CliqzHumanWeb.tmult) {
delete CliqzHumanWeb.docCache[key];
}
}
},
cleanHttpCache: function() {
for(var key in CliqzHumanWeb.httpCache) {
if ((CliqzHumanWeb.counter - CliqzHumanWeb.httpCache[key]['time']) > 60*CliqzHumanWeb.tmult) {
delete CliqzHumanWeb.httpCache[key];
}
}
for(var key in CliqzHumanWeb.httpCache401) {
if ((CliqzHumanWeb.counter - CliqzHumanWeb.httpCache401[key]['time']) > 60*CliqzHumanWeb.tmult) {
delete CliqzHumanWeb.httpCache401[key];
}
}
},
getHeaders: function(strData) {
var o = {};
o['status'] = strData.split(" ")[1];
var l = strData.split("\n");
for(var i=0;i<l.length;i++) {
if (l[i].indexOf('Location: ') == 0) {
o['loc'] = decodeURIComponent(l[i].split(" ")[1].trim());
}
if (l[i].indexOf('WWW-Authenticate: ') == 0) {
var tmp = l[i].split(" ");
var tmp = tmp.slice(1, tmp.length).join(" ");
o['auth'] = tmp;
}
}
return o;
},
httpObserver: {
// check the non 2xx page and report if this is one of the cliqz result
observeActivity: function(aHttpChannel, aActivityType, aActivitySubtype, aTimestamp, aExtraSizeData, aExtraStringData) {
try {
var aChannel = aHttpChannel.QueryInterface(nsIHttpChannel);
// Return if it's a private tab.
if(aChannel.isChannelPrivate != undefined && aChannel.isChannelPrivate) return;
var url = decodeURIComponent(aChannel.URI.spec);
var ho = CliqzHumanWeb.getHeaders(aExtraStringData);
var status = ho['status'];
var loc = ho['loc'];
var httpauth = ho['auth'];
if(gadurl.test(url) && aChannel && aChannel.referrer) {
var refU = aChannel.referrer.spec;
CliqzHumanWeb.linkCache[url] = {'s': ''+refU, 'time': CliqzHumanWeb.counter};
}
if (status=='301' || status == '302') {
CliqzHumanWeb.httpCache[url] = {'status': status, 'time': CliqzHumanWeb.counter, 'location': loc};
}
else if (status=='401') {
CliqzHumanWeb.httpCache401[url] = {'time': CliqzHumanWeb.counter};
}
else if(status){
CliqzHumanWeb.httpCache[url] = {'status': status, 'time': CliqzHumanWeb.counter};
}
} catch(ee){};
}
},
historyTimeFrame: function(callback) {
Cu.import('resource://gre/modules/PlacesUtils.jsm');
var history = [];
var min, max;
var res = [];
var st = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase).DBConnection.createStatement("SELECT min(last_visit_date) as min_date, max(last_visit_date) as max_date FROM moz_places");
var res = [];
st.executeAsync({
handleResult: function(aResultSet) {
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
res.push({"minDate": row.getResultByName("min_date"), "maxDate": row.getResultByName("max_date")});
}
},
handleError: function(aError) {
_log("SQL error: " + aError.message);
callback(true);
},
handleCompletion: function(aReason) {
if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
_log("SQL canceled or aborted");
callback(null);
}
else {
try {
min = parseInt(res[0]['minDate'] / 1000);
max = parseInt(res[0]['maxDate'] / 1000);
} catch (ex) {}
callback(min, max);
}
}
});
},
SQL: function(sql, onRow, callback, parameters) {
// temporary fix to avoid console logs if human web is disabled
// the history listner should be handled better if HW module is disabled
if(!CliqzHumanWeb.dbConn) return;
var st = CliqzHumanWeb.dbConn.createAsyncStatement(sql);
for(var key in parameters) {
st.params[key] = parameters[key];
}
CliqzHumanWeb._SQL(CliqzHumanWeb.dbConn, st, onRow, callback);
},
_SQL: function(dbConn, statement, onRow, callback) {
statement.executeAsync({
onRow: onRow,
callback: callback,
handleResult: function(aResultSet) {
var resultCount = 0;
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
resultCount++;
if (this.onRow) {
this.onRow(statement.row);
}
}
if (this.callback) {
this.callback(resultCount);
}
},
handleError: function(aError) {
_log("Error (" + aError.result + "):" + aError.message);
if (this.callback) {
this.callback(0);
}
},
handleCompletion: function(aReason) {
// Always called when done
}
});
statement.finalize();
},
deleteVisit: function(url) {
CliqzHumanWeb.SQL("delete from usafe where url = :url", null, null, {
url: CliqzHumanWeb.escapeSQL(url)
});
},
deleteTimeFrame: function() {
CliqzHumanWeb.historyTimeFrame(function(min, max) {
CliqzHumanWeb.SQL("delete from usafe where last_visit < :min", null, null, {
min: min
});
CliqzHumanWeb.SQL("delete from usafe where last_visit > :max", null, null, {
max: max
});
});
},
clearHistory: function() {
CliqzHumanWeb.SQL("delete from usafe");
},
historyObserver: {
onBeginUpdateBatch: function() {},
onEndUpdateBatch: function() {
CliqzHumanWeb.deleteTimeFrame();
},
onVisit: function(aURI, aVisitID, aTime, aSessionID, aReferringID, aTransitionType) {},
onTitleChanged: function(aURI, aPageTitle) {},
onBeforeDeleteURI: function(aURI) {},
onDeleteURI: function(aURI) {
CliqzHumanWeb.deleteVisit(aURI.spec);
},
onClearHistory: function() {
CliqzHumanWeb.clearHistory();
},
onPageChanged: function(aURI, aWhat, aValue) {},
onDeleteVisits: function() {},
QueryInterface: XPCOMUtils.generateQI([Ci.nsINavHistoryObserver])
},
linkCache: {},
cleanLinkCache: function() {
for(var key in CliqzHumanWeb.linkCache) {
if ((CliqzHumanWeb.counter - CliqzHumanWeb.linkCache[key]['time']) > 30*CliqzHumanWeb.tmult) {
delete CliqzHumanWeb.linkCache[key];
}
}
},
getRedirects: function(url, _res) {
var res = _res || []
try{
for(var key in CliqzHumanWeb.httpCache) {
if(CliqzHumanWeb.httpCache[key]){
if (CliqzHumanWeb.httpCache[key]['location']!=null && (CliqzHumanWeb.httpCache[key]['status']=='301' || CliqzHumanWeb.httpCache[key]['status']=='302')) {
if (CliqzHumanWeb.httpCache[key]['location']==url || decodeURIComponent(CliqzHumanWeb.httpCache[key]['location']) == url) {
res.unshift(key)
CliqzHumanWeb.getRedirects(key, res);
}
}
}
}
}
catch(ee){};
return res;
},
userSearchTransition: function(rq){
// now let's manage the userTransitions.search
if (rq!=null) {
var source = rq['qurl'];
var query = rq['q'].replace(/\s+/g, " ").trim().toLowerCase();
if (source && query && query.length>0) {
// we have both the source and the query,
// let's see if we have done the query
if (CliqzHumanWeb.userTransitions['search'][query] == null) {
CliqzHumanWeb.userTransitions['search'][query] = {'time': CliqzHumanWeb.counter, 'data': []}
}
CliqzHumanWeb.userTransitions['search'][query]['data'].push([source, CliqzHumanWeb.counter - CliqzHumanWeb.userTransitions['search'][query]['time']]);
}
}
},
getParametersQS: function(url) {
var res = {};
var KeysValues = url.split(/[\?&]+/);
for (var i = 0; i < KeysValues.length; i++) {
var kv = KeysValues[i].split("=");
if (kv.length==2) res[kv[0]] = kv[1];
}
return res;
},
getEmbeddedURL: function(targetURL) {
var ihttps = targetURL.lastIndexOf('https://')
var ihttp = targetURL.lastIndexOf('http://')
if (ihttps>0 || ihttp>0) {
// contains either http or https not ont he query string, very suspicious
var parqs = CliqzHumanWeb.getParametersQS(targetURL);
if (parqs['url']) {
return decodeURIComponent(parqs['url']);
}
}
else return null;
},
auxIsAlive: function() {
return true;
},
auxGetPageData: function(url, page_data, original_url, onsuccess, onerror) {
var error_message = null;
var req = Components.classes['@mozilla.org/xmlextras/xmlhttprequest;1'].createInstance();
/*
We need a try catch block here, because there are some URLs which throw malformed URI error,
hence stalling the double fetch on the same row.
Such URLs should not be there at the first place, but in-case they are, we set them as private.
*/
try{
req.open('GET', url, true);
}catch(ee){
onerror(url, page_data, original_url, error_message);
return;
}
req.overrideMimeType('text/html');
req.channel.loadFlags |= Ci.nsIRequest.LOAD_ANONYMOUS;
//req.withCredentials = false;
//req.setRequestHeader("Authorization", "true");
req.onload = function(){
if (req.status != 200 && req.status != 0 /* local files */){
error_message = 'status not valid: ' + req.status;
_log("Error on doublefetch: " + error_message);
req.onerror();
}
else {
// there has been a redirect, we cannot guarantee that cookies were
// not sent, therefore fail and consider as private
if (req.responseURL != url) {
if (decodeURI(decodeURI(req.responseURL)) != decodeURI(decodeURI(url))) {
error_message = 'dangerous redirect';
_log("Error on doublefetch: " + error_message);
_log("DANGER: " + url + ' ' + req.responseURL);
req.onerror();
return;
}
}
var hiddenWindow = Services.appShell.hiddenDOMWindow;
var parser = new hiddenWindow.DOMParser();
var doc = parser.parseFromString(req.responseText, "text/html");
var x = CliqzHumanWeb.getPageData(url, doc);
CliqzHumanWeb.docCache[url] = {'time': CliqzHumanWeb.counter, 'doc': doc};
onsuccess(url, page_data, original_url, x);
}
}
req.onerror = function() {
onerror(url, page_data, original_url, error_message);
}
req.ontimeout = function() {
error_message = 'timeout';
_log("Error on doublefetch: " + error_message);
req.onerror();
}
req.timeout = 10000;
req.send(null);
},
auxIntersection: function(a, b) {
var ai=0, bi=0;
var result = new Array();
while( ai < a.length && bi < b.length ) {
if (a[ai] < b[bi] ){ ai++; }
else if (a[ai] > b[bi] ){ bi++; }
else {
result.push(a[ai]);
ai++;
bi++;
}
}
return result;
},
auxUnion: function(a, b) {
var h = {};
for (var i = a.length-1; i >= 0; -- i) h[a[i]] = a[i];
for (var i = b.length-1; i >= 0; -- i) h[b[i]] = b[i];
var res = [];
for (var k in h) {
if (h.hasOwnProperty(k)) res.push(h[k]);
}
return res;
},
validDoubleFetch: function(struct_bef, struct_aft, options) {
// compares the structure of the page when rendered in Firefox with the structure of
// the page after.
_log("xbef: " + JSON.stringify(struct_bef));
_log("xaft: " + JSON.stringify(struct_aft));
// Check if struct_bef or struct_aft is not null, in case anyone is then we mark it as private.
// if any of the titles is null (false), then decline (discard)
if (!(struct_bef && struct_aft)) {
_log("fovalidDoubleFetch: found an empty structure");
return false;
}
if (!(struct_bef['t'] && struct_aft['t'])) {
_log("fovalidDoubleFetch: found an empty title");
return false;
}
// if any of the two struct has a iall to false decline
if (!(struct_bef['iall'] && struct_aft['iall'])) {
_log("fovalidDoubleFetch: found a noindex");
return false;
}
// check that there are not different number of frames (or iframes) with an internal
// link on the two different loads (with and with session), if so, the frame (iframe)
// might contain a password field. Cannot afford to fetch all iframes on page. Only
// internal are considered, externals are likely to vary a lot due to advertisement.
//
/*
Adding key to check how many pages will we loose if frame check is turned on
if (struct_bef['nfsh']==null || struct_aft['nfsh']==null || struct_bef['nfsh']!=struct_aft['nfsh']) {
_log("fovalidDoubleFetch: number of internal frames does not match");
return false;
}
if (struct_bef['nifsh']==null || struct_aft['nifsh']==null || struct_bef['nifsh']!=struct_aft['nifsh']) {
_log("fovalidDoubleFetch: number of internal iframes does not match");
return false;
}
*/
if (struct_bef['canonical_url'] != struct_aft['canonical_url']) {
// if canonicals are different, in principle are different pages,
if (struct_aft['canonical_url']!=null && struct_aft['canonical_url'].length>0 && struct_bef['canonical_url']==null) {
// unless in the case that struct_aft (after) has a canonical different than null
// and struct_bef has no canonical,
}
else {
return false;
}
}
if (options.structure_strict==true) {
// if there is enough html length, do the ratio, if below or above 10% then very imbalance, discard
var length_html_ok = true;
var length_text_ok = true;
var ratio_lh = (struct_bef['lh'] || 0) / ((struct_bef['lh'] || 0) + (struct_aft['lh'] || 0));
if ((struct_bef['lh'] || 0) > 10*1024) {
var ratio_lh = (struct_bef['lh'] || 0) / ((struct_bef['lh'] || 0) + (struct_aft['lh'] || 0));
if (ratio_lh < 0.10 || ratio_lh > 0.90) {
_log("fovalidDoubleFetch: lh is not balanced");
length_html_ok = false;
}
}
// if there is enough html length, do the ratio, if below or above 10% then very imbalance, discard
var ratio_nl = (struct_bef['nl'] || 0) / ((struct_bef['nl'] || 0) + (struct_aft['nl'] || 0));
if ((struct_bef['lh'] || 0) > 30) {
var ratio_nl = (struct_bef['nl'] || 0) / ((struct_bef['nl'] || 0) + (struct_aft['nl'] || 0));
if (ratio_nl < 0.10 || ratio_nl > 0.90) {
_log("fovalidDoubleFetch: nl is not balanced");
length_text_ok = false;
}
}
if (!length_text_ok && !length_html_ok) return false;
}
// check for passwords and forms if there is no canonical on the after (double fetched with no session)
if (struct_aft['canonical_url']==null || struct_aft['canonical_url']=='') {
// if had no password inputs before and it has after, decline
if ((struct_bef['nip'] == null || struct_aft['nip'] == null) || (struct_bef['nip'] == 0 && struct_aft['nip'] > 0)) {
_log("validDoubleFetch: fail nip");
return false;
}
// if had no forms before and it has after, decline
if ((struct_bef['nf'] == null || struct_aft['nf'] == null) || (struct_bef['nf'] == 0 && struct_aft['nf'] > 0)) {
_log("validDoubleFetch: fail text nf");
return false;
}
}
// compare that titles are equal, if not equal, use the jaccard coefficient, decline if <=0.5
var t1 = struct_bef['t'] || '';
var t2 = struct_aft['t'] || '';
var jc = 1.0;
if (t1!=t2) {
var vt1 = t1.split(' ').filter(function(el) {return el.length>1});
var vt2 = t2.split(' ').filter(function(el) {return el.length>1});
jc = CliqzHumanWeb.auxIntersection(vt1,vt2).length / CliqzHumanWeb.auxUnion(vt1,vt2).length;
if (Math.max(vt1.length, vt2.length) <= 4) {
// the longest titles is 4 tokens long, the, we are a bit flexible on title differences
if (jc >= 0.5) return true;
else {
_log("short title fail title overlap");
return false;
}
}
else {
// the longest titles has 4 or more tokens, be more restrictive
if (jc <= 0.5) {
// one last check, perhaps it's an encoding issue
var tt1 = t1.replace(/[^A-Za-z 0-9 \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~]*/g, '');
var tt2 = t2.replace(/[^A-Za-z 0-9 \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~]*/g, '');
if ((tt1.length > t1.length*0.5) && ((tt2.length > t2.length*0.5))) {
// if we have not decreased the titles by more than 50%
var vtt1 = tt1.split(' ').filter(function(el) {return el.length>1});
var vtt2 = tt2.split(' ').filter(function(el) {return el.length>1});
jc = CliqzHumanWeb.auxIntersection(vtt1,vtt2).length / CliqzHumanWeb.auxUnion(vtt1,vtt2).length;
// we are more demanding on the title overlap now
if (jc <= 0.80) {
_log("validDoubleFetch: fail title overlap after ascii");
return false;
}
}
else {
_log("validDoubleFetch: fail title overlap");
return false;
}
}
// if the titles are not a perfect match then check for more structural things like number of inputs
// that are type password and number of forms. This is prone to false positives because when not logged in
// legitimate sites something prompt you to register
// if had no password inputs before and it has after, decline
if ((struct_bef['nip'] == null || struct_aft['nip'] == null) || (struct_bef['nip'] == 0 && struct_aft['nip'] > 0)) {
_log("validDoubleFetch: fail nip");
return false;
}
// if had no forms before and it has after, decline
if ((struct_bef['nf'] == null || struct_aft['nf'] == null) || (struct_bef['nf'] == 0 && struct_aft['nf'] > 0)) {
_log("validDoubleFetch: fail text nf");
return false;
}
return true;
}
}
else {
// exactly same title
return true;
}
_log("validDoubleFetch: default option");
return false;
},
getCleanerURL: function(url) {
var clean_url = url;
// check first if there is a query string,
var url_parts = CliqzHumanWeb.parseURL(url);
if (!url_parts) return null;
if (url_parts && url_parts.query_string && url_parts.query_string != '') {
// it has a query string, either by ? # or ;
clean_url = url_parts.protocol + '://' + url_parts.hostname + url_parts.path;
}
else {
// it has neither query_string or hash or semicolon, so let's try to remove the last segment
if (url_parts && url_parts.path && url_parts.path != '') {
var qs = url_parts.path.split('/')
var cqs = [];
for (let i=0;i<qs.length;i++) if (qs[i]!='') cqs.push(qs[i]);
if (cqs.length >= 3) {
// more than 3, /a/b/c, let's remove the last one
var new_path = cqs.slice(0, cqs.length-1).join('/');
clean_url = url_parts.protocol + '://' + url_parts.hostname + '/' + new_path + '/';
}
}
}
if (clean_url != url) {
// they are different, sanity checks
if (CliqzHumanWeb.isSuspiciousURL(clean_url) || CliqzHumanWeb.dropLongURL(clean_url)) return url;
else return clean_url;
}
else return url;
},
fetchReferral: function(referral_url, callback) {
_log("PPP in fetchReferral: " + referral_url);
if (referral_url && referral_url!='') {
if (CliqzHumanWeb.docCache[referral_url]==null) {
CliqzHumanWeb.auxGetPageData(referral_url, null, null, function(referral_url) {
_log("PPP in fetchReferral success auxGetPageData: " + referral_url);
callback();
},
function(referral_url) {
_log("PPP in fetchReferral failure auxGetPageData: " + referral_url);
callback();
});
}
else {
_log("PPP in fetchReferral already in docCache: " + referral_url);
callback();
}
}
else callback();
},
doubleFetch: function(url, page_doc) {
// Need to add check for page MU.
// one last validation whether should be fetchable or not. If we cannot send that URL because it's
// private/suspicious/search_result page/etc. we can mark it as private directly
var isok = true;
if (page_doc ==null || page_doc['x'] == null) {
// this should not happen, but it does. Need to debug why the 'x' field gets lost
// right now, let's set is a private to avoid any risk
//
isok = false
}
if (page_doc && page_doc['x'] && page_doc['x']['iall'] == false) {
// the url is marked as noindex
isok = false;
}
// the url is suspicious, this should never be the case here but better safe
//
if (CliqzHumanWeb.isSuspiciousURL(url) == true) isok = false;
if (CliqzHumanWeb.dropLongURL(url)) {
_log("The url: " + url + " is long");
if (page_doc && page_doc['x'] && page_doc['x']['canonical_url']) {
// the url is to be drop, but it has a canonical URL so it should be public
if (CliqzHumanWeb.dropLongURL(page_doc['x']['canonical_url'])) {
// wops, the canonical is also bad, therefore mark as private
_log("The canonical url: " + page_doc['x']['canonical_url'] + " is also long");
isok = false;
}
else {
// there we are in the good scenario in which canonical looks ok although
// url did not
isok = true;
}
}
else {
if(page_doc['isMU']){
isok = true;
}
else{
isok = false;
}
}
}
if (isok) {
_log("going to double-fetch: " + url);
CliqzHumanWeb.auxGetPageData(url, page_doc, url, function(url, page_doc, original_url, data) {
// data contains the public data of the url double-fetch,
_log("success on doubleFetch, need further validation" + url);
if (CliqzHumanWeb.validDoubleFetch(page_doc['x'], data, {'structure_strict': false})) {
//
// If the double fetch is validated, we will now check for the iFrame and frameSets
// Since we only need the telemetry for now, this seems to be the right place
// we need to inject the page structure. The final event will have an extra key
// nifshmatch : true / false , nfshmatch: true / false.,
// nifshbf : Iframe count in struct_bef, nifshbf : framsetcount in before.
// This key is added to both page_doc and data.
//
let nifshmatch = CliqzHumanWeb.validFrameCount(page_doc['x'], data);
let nfshmatch = CliqzHumanWeb.validFrameSetCount(page_doc['x'], data);
data.nifshmatch = nifshmatch;
data.nfshmatch = nfshmatch;
data.nifshbf = page_doc.x.nifsh;
data.nfshbf = page_doc.x.nifsh;
//
// url, we should have the data of the double for the referral in CliqzHumanWeb.docCache
//
CliqzHumanWeb.fetchReferral(page_doc['ref'], function() {
var strict_value = CliqzHumanWeb.calculateStrictness(url, page_doc);
if (page_doc['ref'] && page_doc['ref']!='') {
// the page has a referral
_log("PPP: page has a referral, " + url + " < " + page_doc['ref']);
var hasurl = CliqzHumanWeb.hasURL(page_doc['ref'], url);
_log("PPP: page has a referral, " + url + " < " + page_doc['ref'] + ">>>> " + hasurl);
// overwrite strict value because the link exists on a public fetchable page
if (hasurl) strict_value = false;
}
else {
// page has no referral
_log("PPP: page has NO referral, " + url);
// we do not know the origin of the page, run the dropLongURL strict version, if
// there is no canonical or if there is canonical and is the same as the url,
}
_log("strict URL:" + url + " > " + strict_value);
if (CliqzHumanWeb.validDoubleFetch(page_doc['x'], data, {'structure_strict': strict_value})) {
// we do not know the origin of the page, run the dropLongURL strict version
if (CliqzHumanWeb.dropLongURL(url, {'strict': strict_value})) {
if (page_doc && page_doc['x'] && page_doc['x']['canonical_url']) {
if (CliqzHumanWeb.dropLongURL(page_doc['x']['canonical_url'], {'strict': strict_value})) {
_log("doubleFetch failed on dropLongURL strict=true:" + url);
CliqzHumanWeb.setAsPrivate(url);
return;
}
}
else {
_log("doubleFetch failed on dropLongURL strict=true:" + url);
CliqzHumanWeb.setAsPrivate(url);
return;
}
}
}
else {
// the strict version of validDoubleFetch fails for a page with no referral,
// since we do not know the origin mark as private
_log("doubleFetch failed on structure_strict=true: " + url);
CliqzHumanWeb.setAsPrivate(url);
return;
}
_log("success on doubleFetch, need further validation");
//
// we need to modify the 'x' field of page_doc to substitute any structural information about
// the page content by the data coming from the doubleFetch (no session)
//
var first_url_double_fetched = url;
//
// we need to modify the 'x' field of page_doc to substitute any structural information about
// the page content by the data coming from the doubleFetch (no session)
//
// at this point we might have 3 different urls: url, page_doc['x']['canonical_url'] (either
// one of them has to have passed the dropLongURL test), and finally we also have data['canonical_url']
// (which has not passed the dropLongURL test). In principle, data['canonical_url']==page_doc['x']['canonical_url']
// but is not always the case, different calls can give different urls
if (data['canonical_url']!=null && data['canonical_url'] != '' && CliqzHumanWeb.dropLongURL(data['canonical_url'])==false) {
page_doc['url'] = data['canonical_url'];
page_doc['x'] = data;
}
else {
if (page_doc['x']['canonical_url']!=null && page_doc['x']['canonical_url'] != '' && CliqzHumanWeb.dropLongURL(page_doc['x']['canonical_url'])==false) {
page_doc['url'] = page_doc['x']['canonical_url'];
page_doc['x'] = data;
page_doc['x']['canonical_url'] = page_doc['url'];
}
else {
// there was no canonical either on page_doc['x'] or in data or it was droppable
if (CliqzHumanWeb.dropLongURL(url)==false) {
page_doc['url'] = url;
page_doc['x'] = data;
if (page_doc['x']['canonical_url']!=null && page_doc['x']['canonical_url'] != '') {
page_doc['x']['canonical_url'] = url;
}
}
else {
// this should not happen since it would be covered by the isok checks, but better safe,
CliqzHumanWeb.setAsPrivate(url);
return;
}
}
}
var clean_url = CliqzHumanWeb.getCleanerURL(page_doc['url']);
if (clean_url != page_doc['url']) {
// we have a candidate for a cleaner url (without query_string) or without the last segment
// of the path, we want to double fetch it and if successful, then, we could replace the url
// because it would be cleaner hence safer
//
_log("going to clean_url double-fetch: " + clean_url);
CliqzHumanWeb.auxGetPageData(clean_url, page_doc, first_url_double_fetched, function(url, page_doc, original_url, data) {
_log("success on clean_url doubleFetch, need further validation");
if (CliqzHumanWeb.validDoubleFetch(page_doc['x'], data, {'structure_strict': false})) {
// if it the second double fetch is valid, that means that the clean_url is (url parameter) is
// equivalent, so we can replace
page_doc['url'] = url;
page_doc['x'] = data;
CliqzHumanWeb.setAsPublic(original_url);
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});
}
else {
// the page with the clean_urls does not return the same, it can be two cases here, one is that
// the content is just totally different, in this case, we should send the page with the unclean_url,
// the other case is that the content is different but now we have passwords where we did not have before,
// in such a case, it's safer to assume that the fragments cleaned were identifiying a user, and the
// website is redirecting to the login page, in such a case, we should not send the page at all, in fact, we
// should mark it as private just to be sure,
if (CliqzHumanWeb.debug) {
_log("checking clean_url, page_doc: " + JSON.stringify(page_doc));
_log("checking clean_url, data: " + JSON.stringify(data));
}
if (page_doc['x']['nip'] < data['nip']) {
// the page with url_clean have more input password fields or more forms, this is dangerous,
_log("failure on checking the password and forms for clean_url: " + original_url + " set as private");
CliqzHumanWeb.setAsPrivate(original_url);
}
else {
// safe, here we will send the url before clean_url
CliqzHumanWeb.setAsPublic(original_url);
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});
}
}
},
function(url, page_doc, original_url, error_message) {
_log("failure on clean_url doubleFetch! " + "structure did not match");
// there was a failure, the clean_url does not go to the same place, therefore it's better
// not to replace
CliqzHumanWeb.setAsPublic(original_url);
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});
});
}
else {
CliqzHumanWeb.setAsPublic(original_url);
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});
}
});
}
else {
_log("failure on doubleFetch! " + "structure did not match");
CliqzHumanWeb.setAsPrivate(url);
}
},
function(url, page_doc, original_url, error_message) {
_log("failure on doubleFetch! " + error_message);
CliqzHumanWeb.setAsPrivate(url);
});
}
else {
_log("doubleFetch refused to process this url: " + url);
CliqzHumanWeb.setAsPrivate(url);
}
},
hasURL: function(source_url, target_url) {
// the target_url is in the source_url
try {
var tt = CliqzHumanWeb.docCache[source_url];
if (tt) {
var cd = tt['doc'];
if (!cd) {
// fetch the content of the source_url,
//
_log("hasURL no CD!!! ");
return false;
}
}
else return false;
var target_url_no_protocol = target_url.replace(/^http(s?)\:\/\//, '');
var target_url_relative = null;
try {
var source_hostname = CliqzHumanWeb.parseURL(source_url).hostname;
var target_hostname = CliqzHumanWeb.parseURL(target_url).hostname;
} catch(ee) {
return false;
}
if (source_hostname == target_hostname) {
// same domain, path could be relative
target_url_relative = '/';
var v = target_url_no_protocol.split('/');
if (v.length>1) target_url_relative = '/' + v.slice(1, v.length).join('/')
}
//var html = cd.documentElement.innerHTML;
//var ind1 = html.indexOf(target_url_no_protocol);
var found = false;
var links = cd.documentElement.getElementsByTagName('a');
for(var i=0;i<links.length;i++) {
try {
var link = links[i].href;
link = link.replace(/^http(s?)\:\/\//, '');
if (link == target_url_no_protocol) {
found = true;
break;
}
else {
if (target_url_relative && link == target_url_relative) {
found = true;
break;
}
}
} catch(ee) {}
}
return found;
} catch(ee) {
_log("Error on hasURL: " + ee);
return false;
}
},
/*
getMetaRefresh: function(cd, url){
var metas = null;
var redURL = null;
var title = null;
try{redURL = cd.split('URL=')[1].split('>')[0].replace('"','').replace("'","")}catch(ee){};
CliqzHumanWeb.httpCache[url] = {'status': '301', 'time': CliqzHumanWeb.counter, 'location': redURL};
//Get first redirection.. for yahoo and stuff
if(url.indexOf('r.search.yahoo.com') > -1){
try{var _url = CliqzHumanWeb.linkCache[decodeURIComponent(url)]['s']}catch(ee){var _url = url}
CliqzHumanWeb.linkCache[decodeURIComponent(redURL.replace("'",""))] = {'s': ''+_url, 'time': CliqzHumanWeb.counter};
}
return redURL;
},
*/
eventDoorWayPage: function(cd){
var payload = {};
var url = cd.location.href;
var doorwayURL = cd.getElementsByTagName('a')[0].href;
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){};
var orignalDomain = CliqzHumanWeb.parseURL(url).hostname;
var dDomain = CliqzHumanWeb.parseURL(doorwayURL).hostname;
if(orignalDomain == dDomain) return;
payload = {"url":url, "durl":doorwayURL,"ctry": location};
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'doorwaypage', 'payload': payload});
},
getPageData: function(url, cd) {
var len_html = null;
var len_text = null;
var title = null;
var numlinks = null;
var inputs = null;
var inputs_nh = null;
var inputs_pwd = null;
var frames_same_host = null;
var iframes_same_host = null;
var frames = null;
var forms = null;
var pg_l = null;
var metas = null;
var tag_html = null;
var iall = true;
var all = null;
var canonical_url = null;
var url_host = null;
var frame_host = null;
try {
url_host = CliqzHumanWeb.parseURL(url).hostname;
} catch(ee) {
url_host = null;
}
try { len_html = cd.documentElement.innerHTML.length; } catch(ee) {}
try { len_text = cd.documentElement.textContent.length; } catch(ee) {}
try { title = cd.getElementsByTagName('title')[0].textContent; } catch(ee) {}
//title = unescape(encodeURIComponent(title));
try { numlinks = cd.getElementsByTagName('a').length; } catch(ee) {}
try {
inputs = cd.getElementsByTagName('input') || [];
inputs_nh = 0;
inputs_pwd = 0;
for(var i=0;i<inputs.length;i++) {
if (inputs[i]['type'] && inputs[i]['type']!='hidden') inputs_nh+=1;
if (inputs[i]['type'] && inputs[i]['type']=='password') inputs_pwd+=1;
}
} catch(ee) {}
try {
frames = cd.getElementsByTagName('frame') || [];
frames_same_host = 0;
for(var i=0;i<frames.length;i++) {
if (frames[i]['src']) {
var tsrc = frames[i]['src'];
if (frames[i]['src'].startsWith('//')) tsrc = 'http:' + frames[i]['src'];
try {
frame_host = CliqzHumanWeb.parseURL(tsrc).hostname;
if (frame_host===url_host || frame_host=='browser') frames_same_host++;
} catch(ee) {
frames_same_host++;
}
}
}
}
catch(ee) {}
try {
frames = cd.getElementsByTagName('iframe') || [];
iframes_same_host = 0;
for(var i=0;i<frames.length;i++) {
if (frames[i]['src']) {
var tsrc = frames[i]['src'];
if (frames[i]['src'].startsWith('//')) tsrc = 'http:' + frames[i]['src'];
try {
frame_host = CliqzHumanWeb.parseURL(tsrc).hostname;
if (frame_host===url_host || frame_host=='browser') iframes_same_host++;
} catch(ee) {
iframes_same_host++;
}
}
}
}
catch(ee) {}
try { forms = cd.getElementsByTagName('form'); } catch(ee) {}
//Detect doorway pages
// TBF : Need to make detecting of doorway page more strong. Currently lot of noise getting through.
/*
if(numlinks == 1 && cd.location){
CliqzHumanWeb.eventDoorWayPage(cd);
}
*/
var metas = cd.getElementsByTagName('meta');
// extract the language of th
try {
for (let i=0;i<metas.length;i++) {if (metas[i].getAttribute("http-equiv") == "content-language" || metas[i].getAttribute("name") == "language") {
pg_l = metas[i].getAttribute("content");
}};
if (pg_l == null) {
tag_html = cd.getElementsByTagName('html');
pg_l = tag_html[0].getAttribute("lang");
};
}catch(ee){}
// extract if indexable, no noindex on robots meta tag
try {
for (let i=0;i<metas.length;i++) {
var cnt = metas[i].getAttribute('content').toLowerCase();
if (cnt!=null && cnt.indexOf('noindex') > -1) {
iall = false;
}
}
}catch(ee){}
// extract the canonical url if available
var link_tag = cd.getElementsByTagName('link');
for (var j=0;j<link_tag.length;j++) {
if (link_tag[j].getAttribute("rel") == "canonical") {
canonical_url = link_tag[j].href;
// This check is done because of misplaces titles on sites like 500px, youtube etc.
// Since could not find a proper fix, hence dropping canonical URL looks like a safe idea.
if(CliqzHumanWeb.can_url_match[canonical_url] && CliqzHumanWeb.can_url_match[canonical_url] != url) canonical_url = null;
}
}
if (canonical_url != null && canonical_url.length > 0) {
// check that is not relative
if (canonical_url[0] == '/') {
try {
var ourl = CliqzHumanWeb.parseURL(url);
// ignore if httpauth or if non standard port
canonical_url = ourl['protocol'] + '://' + ourl['hostname'] + canonical_url;
} catch(ee) {}
}
}
// extract the location of the user (country level)
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){}
var x = {'lh': len_html, 'lt': len_text, 't': title, 'nl': numlinks, 'ni': (inputs || []).length, 'ninh': inputs_nh, 'nip': inputs_pwd, 'nf': (forms || []).length, 'pagel' : pg_l , 'ctry' : location, 'iall': iall, 'canonical_url': canonical_url, 'nfsh': frames_same_host, 'nifsh': iframes_same_host};
return x;
},
getHTML(url) {
// lazy load fallback urls
const possibleUrls = [
url => decodeURI(decodeURI(url)),
url => decodeURIComponent(url),
];
function getDoc(url) {
return getHTML(url).then( docs => {
const doc = docs[0];
if (doc) {
return doc;
} else {
throw "not found";
}
}).catch( () => {
const possibleUrl = possibleUrls.shift();
if (possibleUrl) {
const url = possibleUrl();
return getDoc(url);
} else {
throw "not found";
}
});
}
return getDoc(url);
},
createDoc(html) {
const hiddenWindow = Services.appShell.hiddenDOMWindow;
const parser = new hiddenWindow.DOMParser();
return parser.parseFromString(html, "text/html");
},
getCD(url) {
return this.getHTML(url).then( html => {
const doc = this.createDoc(html);
// monkey patching the doc so it looks like regular document element
/*
doc.location = {
href: url
};
*/
return doc;
});
},
captureJSRefresh: function(aRequest, aURI){
if(aRequest && aRequest.referrer) {
var refU = aRequest.referrer.asciiSpec;
var newURL = aURI.spec;
if(refU.indexOf('t.co/') > -1) CliqzHumanWeb.httpCache[refU] = {'status': '301', 'time': CliqzHumanWeb.counter, 'location': aURI.spec};
//Get first redirection.. for yahoo and stuff
var refyahoo = /\.search.yahoo\..*RU=/;
if(refyahoo.test(refU)){
try{var _url = CliqzHumanWeb.linkCache[decodeURIComponent(refU)]['s']}catch(ee){var _url = refU}
CliqzHumanWeb.linkCache[newURL] = {'s': ''+_url, 'time': CliqzHumanWeb.counter};
}
}
},
listener: {
tmpURL: undefined,
QueryInterface: XPCOMUtils.generateQI(["nsIWebProgressListener", "nsISupportsWeakReference"]),
onLocationChange: function(aProgress, aRequest, aURI) {
// New location, means a page loaded on the top window, visible tab
// Return if it's a private tab.
if(aRequest && aRequest.isChannelPrivate !== undefined && aRequest.isChannelPrivate) {
return;
}
if(aProgress.isLoadingDocument){
CliqzHumanWeb.captureJSRefresh(aRequest, aURI);
}
if (aURI.spec == this.tmpURL) return;
this.tmpURL = aURI.spec;
// This code looks obselete now, will remove in 1+ release.
if (CliqzHumanWeb.ismRefresh){
try{
var tabID = CliqzHumanWeb.getTabID();
if(tabID){
var mrefreshUrl = CliqzHumanWeb.mRefresh[tabID];
var parentRef = CliqzHumanWeb.linkCache[mrefreshUrl]['s'];
CliqzHumanWeb.linkCache[decodeURIComponent(aURI.spec)] = {'s': ''+mrefreshUrl, 'time': CliqzHumanWeb.counter};
CliqzHumanWeb.state['v'][mrefreshUrl]['qr'] = CliqzHumanWeb.state['v'][parentRef]['qr'];
if(CliqzHumanWeb.state['v'][mrefreshUrl]['qr']){
//Change type to ad, else might create confusion.
CliqzHumanWeb.state['v'][mrefreshUrl]['qr']['t'] = 'gad';
}
CliqzHumanWeb.ismRefresh = false;
delete CliqzHumanWeb.mRefresh[tabID];
}
}
catch(ee){};
}
// here we check if user ignored our results and went to google and landed on the same url
var requery = /\.google\..*?[#?&;]q=[^$&]+/; // regex for google query
var yrequery = /.search.yahoo\..*?[#?&;]p=[^$&]+/; // regex for yahoo query
var brequery = /\.bing\..*?[#?&;]q=[^$&]+/; // regex for yahoo query
var reref = /\.google\..*?\/(?:url|aclk)\?/; // regex for google refurl
var rerefurl = /url=(.+?)&/; // regex for the url in google refurl
// var gadurl = /\.google..*?\/(aclk)\?/;
// suggested by AMO - requires further testing in case of rapid tab switching (orange, blue)
// original:
// var currwin = aProgress.topWindow || CliqzUtils.getWindow().gBrowser.selectedBrowser.contentDocument;
// currwin is not used anymore, since the document is fetched using content script.
// var currwin = aProgress.DOMWindow.top;
// if(!currwin) return; //internal FF page
// This code looks obselete now, will remove in 1+ release.
if(gadurl.test(aURI.spec)){
var tabID = CliqzHumanWeb.getTabID();
if(tabID){
CliqzHumanWeb.ismRefresh = true;//{'status': '301', 'time': CliqzHumanWeb.counter, 'location': decodeURIComponent(CliqzHumanWeb.parseUri(aURI.spec)['queryKey']['adurl'])};
CliqzHumanWeb.mRefresh[tabID] = decodeURIComponent(aURI.spec);
}
}
// var currwin = CliqzUtils.getWindow();
// var _currURL = '' + currwin.gBrowser.selectedBrowser.contentDocument.location;
/*
//This needs to go away. Should get the content from contentDocument, but it is coming as null right now.
if(_currURL.indexOf('t.co/') > -1){
CliqzUtils.httpGet(_currURL,
function(res){
if(res && res.response){
try {
var _metaCD = res.response;
var redURL = CliqzHumanWeb.getMetaRefresh(_metaCD,_currURL );
} catch(e){}
}
}, null, 2000);
}
else if(_currURL.indexOf('r.search.yahoo.com') > -1){
CliqzUtils.httpGet(_currURL,
function(res){
if(res && res.response){
try {
var _metaCD = res.response;
var redURL = CliqzHumanWeb.getMetaRefresh(_metaCD,_currURL );
} catch(e){}
}
}, null, 2000);
}
else if(gadurl.test(_currURL)){
var tabID = CliqzHumanWeb.getTabID();
if(tabID){
CliqzHumanWeb.ismRefresh = true;//{'status': '301', 'time': CliqzHumanWeb.counter, 'location': decodeURIComponent(CliqzHumanWeb.parseUri(_currURL)['queryKey']['adurl'])};
CliqzHumanWeb.mRefresh[tabID] = decodeURIComponent(_currURL);
}
}
*/
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActiveAll = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(aURI.spec);
//Check if the URL is know to be bad: private, about:, odd ports, etc.
if (CliqzHumanWeb.isSuspiciousURL(activeURL)) return;
if (activeURL.indexOf('about:')!=0) {
if (CliqzHumanWeb.state['v'][activeURL] == null) {
//if ((requery.test(activeURL) || yrequery.test(activeURL) || brequery.test(activeURL) ) && !reref.test(activeURL)) {
var se = CliqzHumanWeb.checkSearchURL(activeURL);
if (se > -1) {
CliqzUtils.setTimeout(function(url) {
if (!CliqzHumanWeb) {
return;
}
CliqzHumanWeb.getCD(url).then( doc => {
CliqzHumanWeb.checkURL(doc, url, "normal");
CliqzHumanWeb.queryCache[url] = {
d: 0,
q: CliqzHumanWeb.searchCache[se]['q'],
t: CliqzHumanWeb.searchCache[se]['t']
};
let anonSe = CliqzHumanWeb.checkAnonSearchURL(url);
if(anonSe > -1){
try {
let hostName = CliqzHumanWeb.parseURL(url)['hostname'];
let qurl = "https://" + hostName + "/search?q=" + CliqzHumanWeb.searchCache[se]['q'];
let qObj = {};
qObj['qurl'] = qurl;
qObj['ts'] = Date.now();
qObj['tDiff'] = getRandomIntInclusive(1, 20);
CliqzHumanWeb.strictQueries.push(qObj);
CliqzHumanWeb.saveStrictQueries();
} catch(ee) {}
}
});
}, CliqzHumanWeb.WAIT_TIME, activeURL);
}
var status = null;
if (CliqzHumanWeb.httpCache[activeURL]!=null) {
status = CliqzHumanWeb.httpCache[activeURL]['status'];
}
var referral = null;
var qreferral = null;
if (CliqzHumanWeb.linkCache[activeURL] != null) {
//referral = CliqzHumanWeb.maskURL(CliqzHumanWeb.linkCache[activeURL]['s']);
referral = CliqzHumanWeb.linkCache[activeURL]['s'];
}
//Get redirect chain
var red = [];
red = CliqzHumanWeb.getRedirects(activeURL, red);
if(red.length == 0){
red = null;
}
//Set referral for the first redirect in the chain.
if (red && referral == null) {
var redURL = red[0];
var refURL = CliqzHumanWeb.linkCache[redURL];
if(refURL){
referral = refURL['s'];
}
//Update query cache with the redirected URL
if (CliqzHumanWeb.queryCache[redURL]) {
CliqzHumanWeb.queryCache[activeURL] = CliqzHumanWeb.queryCache[redURL];
}
}
CliqzHumanWeb.state['v'][activeURL] = {'url': activeURL, 'a': 0, 'x': null, 'tin': new Date().getTime(),
'e': {'cp': 0, 'mm': 0, 'kp': 0, 'sc': 0, 'md': 0}, 'st': status, 'c': [], 'ref': referral, 'red':red};
if (referral) {
// if there is a good referral, we must inherit the query if there is one
if (CliqzHumanWeb.state['v'][referral] && CliqzHumanWeb.state['v'][referral]['qr']) {
CliqzHumanWeb.state['v'][activeURL]['qr'] = {}
CliqzHumanWeb.state['v'][activeURL]['qr']['q'] = CliqzHumanWeb.state['v'][referral]['qr']['q'];
CliqzHumanWeb.state['v'][activeURL]['qr']['t'] = CliqzHumanWeb.state['v'][referral]['qr']['t'];
CliqzHumanWeb.state['v'][activeURL]['qr']['d'] = CliqzHumanWeb.state['v'][referral]['qr']['d']+1;
//If the depth is greater then two, we need to check if the ref. is of same domain.
//If not then drop the QR object, else keep it.
if(CliqzHumanWeb.state['v'][activeURL]['qr']['d'] > 2){
delete CliqzHumanWeb.state['v'][activeURL]['qr'];
}
else if(CliqzHumanWeb.state['v'][activeURL]['qr']['d'] == 2){
try {
if(CliqzHumanWeb.parseURL(activeURL)['hostname'] != CliqzHumanWeb.parseURL(referral)['hostname']){
delete CliqzHumanWeb.state['v'][activeURL]['qr'];
}
} catch(ee) {
delete CliqzHumanWeb.state['v'][activeURL]['qr'];
}
}
}
}
// Change the type to gad if the it's coming from an ADV. else can give incorrect clicks on SERP results.
if(red && red.length > 0){
for(var i=0;i<red.length;i++){
if(gadurl.test(red[i]) && CliqzHumanWeb.state['v'][activeURL].hasOwnProperty('qr')){
CliqzHumanWeb.state['v'][activeURL]['qr']['t'] = 'gad';
}
}
}
CliqzUtils.setTimeout(function(currURL) {
// Extract info about the page, title, length of the page, number of links, hash signature,
// 404, soft-404, you name it
//
// we cannot get it directly via
// var cd = currWin.gBrowser.selectedBrowser.contentDocument;
// because during the time of the timeout there can be win or tab switching
//
//var activeURL = CliqzHumanWeb.currentURL();
//if (activeURL != currURL) {}
CliqzHumanWeb.getCD(currURL).then(function (cd) {
var se = CliqzHumanWeb.checkSearchURL(currURL);
if (se == -1){
try {
CliqzHumanWeb.checkURL(cd, currURL,"normal");
} catch (e) {
}
//Check active usage...
CliqzHumanWeb.activeUsage += 1;
}
var x = CliqzHumanWeb.getPageData(currURL, cd);
if (x['canonical_url']) {
CliqzHumanWeb.can_urls[currURL] = x['canonical_url'];
CliqzHumanWeb.can_url_match[x['canonical_url']] = currURL;
}
if (CliqzHumanWeb.state['v'][currURL] != null) {
CliqzHumanWeb.state['v'][currURL]['x'] = x;
}
if (CliqzHumanWeb.queryCache[currURL]) {
CliqzHumanWeb.state['v'][currURL]['qr'] = CliqzHumanWeb.queryCache[currURL];
delete CliqzHumanWeb.queryCache[currURL];
}
if (CliqzHumanWeb.state['v'][currURL] != null) {
CliqzHumanWeb.addURLtoDB(currURL, CliqzHumanWeb.state['v'][currURL]['ref'], CliqzHumanWeb.state['v'][currURL]);
CliqzHumanWeb.queryCache[currURL];
}
}, function () {
if (CliqzHumanWeb.debug) {
_log("CANNOT GET THE CONTENT OF : " + currURL);
}
}).catch( ee => {
_log("Error fetching title and length of page: " + ee + " : " + currURL);
});
}, CliqzHumanWeb.WAIT_TIME, activeURL);
}
else {
// wops, it exists on the active page, probably it comes from a back button or back
// from tab navigation
CliqzHumanWeb.state['v'][activeURL]['tend'] = null;
CliqzEvents.pub('HW-activeURL:', {activeURL});
}
}
},
onStateChange: function(aWebProgress, aRequest, aFlag, aStatus) {
}
},
pacemaker: function() {
var activeURL = CliqzHumanWeb.currentURL();
if (activeURL && (activeURL).indexOf('about:')!=0) {
if ((CliqzHumanWeb.counter - CliqzHumanWeb.lastActive) < 5*CliqzHumanWeb.tmult) {
// if there has been an event on the last 5 seconds, if not do no count, the user must
// be doing something else,
//
try {
CliqzHumanWeb.state['v'][activeURL]['a'] += 1;
} catch(ee) {}
}
}
if ((activeURL==null) && ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % 10 == 0)) {
// this one is for when you do not have the page open, for instance, no firefox but console opened
CliqzHumanWeb.pushAllData();
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % 5 == 0) {
var openPages = CliqzHumanWeb.getAllOpenPages();
var tt = new Date().getTime();
for (var url in CliqzHumanWeb.state['v']) {
if (CliqzHumanWeb.state['v'].hasOwnProperty(url)) {
if (openPages.indexOf(url)==-1) {
// not opened
if (CliqzHumanWeb.state['v'][url]['tend']==null) {
CliqzHumanWeb.state['v'][url]['tend'] = tt;
}
if ((tt - CliqzHumanWeb.state['v'][url]['tend']) > CliqzHumanWeb.deadFiveMts*60*1000) {
// move to "dead pages" after 5 minutes
CliqzHumanWeb.state['m'].push(CliqzHumanWeb.state['v'][url]);
CliqzHumanWeb.addURLtoDB(url, CliqzHumanWeb.state['v'][url]['ref'], CliqzHumanWeb.state['v'][url]);
delete CliqzHumanWeb.state['v'][url];
delete CliqzHumanWeb.queryCache[url];
}
}
else {
// stil opened, do nothing.
if ((tt - CliqzHumanWeb.state['v'][url]['tin']) > CliqzHumanWeb.deadTwentyMts*60*1000) {
// unless it was opened more than 20 minutes ago, if so, let's move it to dead pages
CliqzHumanWeb.state['v'][url]['tend'] = null;
CliqzHumanWeb.state['v'][url]['too_long'] = true;
CliqzHumanWeb.state['m'].push(CliqzHumanWeb.state['v'][url]);
CliqzHumanWeb.addURLtoDB(url, CliqzHumanWeb.state['v'][url]['ref'], CliqzHumanWeb.state['v'][url]);
delete CliqzHumanWeb.state['v'][url];
delete CliqzHumanWeb.queryCache[url];
//_log("Deleted: moved to dead pages after 20 mts.");
//_log("Deleted: moved to dead pages after 20 mts: " + CliqzHumanWeb.state['m'].length);
}
}
}
}
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % 10 == 0) {
if (CliqzHumanWeb.debug) {
_log('Pacemaker: ' + CliqzHumanWeb.counter/CliqzHumanWeb.tmult + ' ' + activeURL + ' >> ' + CliqzHumanWeb.state.id);
}
// Ensuring the dbConn is not null.
var dbConn = CliqzHumanWeb.getDBConn();
CliqzHumanWeb.cleanHttpCache();
CliqzHumanWeb.cleanDocCache();
CliqzHumanWeb.cleanLinkCache();
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % (1*60) == 0) {
// every minute
CliqzHumanWeb.listOfUnchecked(1, CliqzHumanWeb.doubleFetchTimeInSec, null, CliqzHumanWeb.processUnchecks);
CliqzHumanWeb.auxGetQuery();
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % 10 == 0) {
var ll = CliqzHumanWeb.state['m'].length;
if (ll > 0) {
var v = CliqzHumanWeb.state['m'].slice(0, ll);
CliqzHumanWeb.state['m'] = CliqzHumanWeb.state['m'].slice(ll, CliqzHumanWeb.state['m'].length);
/*
for(var i=0;i<v.length;i++) {
CliqzHumanWeb.addURLtoDB(url, CliqzHumanWeb.state['v'][url]['ref'], CliqzHumanWeb.state['v'][url]);
}
*/
}
if(!CliqzHumanWeb.bloomFilter){
CliqzHumanWeb.loadBloomFilter();
}
}
//Load ts config
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % (60 * 20 * 1) == 0) {
if (CliqzHumanWeb.debug) {
_log('Load ts config');
}
CliqzHumanWeb.fetchAndStoreConfig();
CliqzHumanWeb.expireQuorumBloomFilter();
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % (60 * 60 * 1) == 0) {
if (CliqzHumanWeb.debug) {
_log('Check if alive');
}
CliqzHumanWeb.checkActiveUsage();
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % (60 * 20 * 1) == 0) {
if (CliqzHumanWeb.debug) {
_log('Save actions stats');
}
CliqzHumanWeb.saveActionStats();
CliqzHumanWeb.sendActionStatsIfNeeded();
}
CliqzHumanWeb.counter += 1;
},
cleanUserTransitions: function(force) {
for(var query in CliqzHumanWeb.userTransitions['search']) {
if ((force) || ((CliqzHumanWeb.counter - CliqzHumanWeb.userTransitions['search'][query]['time']) > CliqzHumanWeb.userTransitionsSearchSession*CliqzHumanWeb.tmult)) {
// the query session is more than 5 minutes old or we are forcing the event,
// if the condition is met and there are more than two elements in data we
// must create the signal
//
if (CliqzHumanWeb.userTransitions['search'][query]['data'].length > 1) {
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){};
var doc = {'q': query, 'sources': CliqzHumanWeb.userTransitions['search'][query]['data'],'ctry': location};
if (CliqzHumanWeb.debug) {
_log(JSON.stringify(doc,undefined,2));
}
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'userTransition.search', 'payload': doc});
}
delete CliqzHumanWeb.userTransitions['search'][query];
}
}
},
pushAllData: function() {
// force send user Transitions sessions even if not elapsed because the browser is shutting down
CliqzHumanWeb.cleanUserTransitions(true);
var tt = new Date().getTime();
var res = [];
for (var url in CliqzHumanWeb.state['v']) {
if (CliqzHumanWeb.state['v'][url]) res.push(url);
}
for (var i=0; i<res.length; i++) {
// move all the pages to m set
var url = res[i];
if (CliqzHumanWeb.state['v'][url]) {
if (CliqzHumanWeb.state['v'][url]['tend']==null) {
CliqzHumanWeb.state['v'][url]['tend'] = tt;
}
CliqzHumanWeb.addURLtoDB(url, CliqzHumanWeb.state['v'][url]['ref'], CliqzHumanWeb.state['v'][url]);
CliqzHumanWeb.state['m'].push(CliqzHumanWeb.state['v'][url]);
delete CliqzHumanWeb.state['v'][url];
delete CliqzHumanWeb.queryCache[url];
}
}
// send them to telemetry if needed
var ll = CliqzHumanWeb.state['m'].length;
if (ll > 0) {
var v = CliqzHumanWeb.state['m'].slice(0, ll);
CliqzHumanWeb.state['m'] = CliqzHumanWeb.state['m'].slice(ll, CliqzHumanWeb.state['m'].length);
CliqzHumanWeb.pushTelemetry();
}
},
unload: function() {
//Check is active usage, was sent
try {var activeUsageTrk = CliqzUtils.getPref('config_activeUsage', null)} catch(ee){};
// Save action stats
CliqzHumanWeb.saveActionStats();
if(activeUsageTrk){
var tDiff = parseInt((new Date().getTime() - activeUsageTrk) / 1000);
if(tDiff && tDiff > 3600){
CliqzHumanWeb.checkActiveUsage();
}
else{
CliqzUtils.setPref('config_activeUsageCount', CliqzHumanWeb.activeUsage);
}
}
// send all the data
CliqzHumanWeb.pushTelemetry();
CliqzUtils.clearTimeout(CliqzHumanWeb.pacemakerId);
CliqzUtils.clearTimeout(CliqzHumanWeb.trkTimer);
},
unloadAtBrowser: function(){
try {
CliqzHumanWeb.activityDistributor.removeObserver(CliqzHumanWeb.httpObserver);
} catch(e){}
},
currentURL: function() {
var currwin = CliqzUtils.getWindow(), ret = null;
if (currwin && currwin.gBrowser) {
ret = currwin.gBrowser.selectedBrowser.currentURI.spec;
}
return CliqzHumanWeb.cleanCurrentUrl(ret);
},
cleanCurrentUrl: function(url){
try {
url = decodeURIComponent(url.trim());
} catch(ee) {}
if (url!=null || url!=undefined) return url;
else return null;
},
pacemakerId: null,
// load from the about:config settings
captureKeyPressPage: function(ev) {
if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['keypresspage']|0)) > 1 * CliqzHumanWeb.tmult) {
if (CliqzHumanWeb.debug) {
//_log('captureKeyPressPage');
}
CliqzHumanWeb.lastEv['keypresspage'] = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
CliqzHumanWeb.state['v'][activeURL]['e']['kp'] += 1;
}
}
},
captureMouseMovePage: function(ev) {
if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['mousemovepage']|0)) > 1 * CliqzHumanWeb.tmult) {
if (CliqzHumanWeb.debug) {
_log('captureMouseMovePage');
}
CliqzHumanWeb.lastEv['mousemovepage'] = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
CliqzHumanWeb.state['v'][activeURL]['e']['mm'] += 1;
}
}
},
getURLFromEvent: function(ev) {
try {
if (ev.target.href!=null || ev.target.href!=undefined) {
return decodeURIComponent(''+ev.target.href);
}
else {
if (ev.target.parentNode.href!=null || ev.target.parentNode.href!=undefined) {
return decodeURIComponent(''+ev.target.parentNode.href);
}
}
}
catch(ee) {
if (CliqzHumanWeb.debug) {
_log('Error in getURLFromEvent: ' + ee);
}
}
return null;
},
contextFromEvent: null,
setContextFromEvent: function(ev) {
try {
var tar = ev.target;
var found = false;
var count = 0;
var def_html = null;
while(!found) {
var html = tar.innerHTML;
if (html.indexOf('http://')!=-1 || html.indexOf('https://')!=-1) {
found = true;
def_html = html;
break;
}
tar = tar.parentNode;
count+=1;
if (count > 4) break;
}
if (found && def_html) {
CliqzHumanWeb.contextFromEvent = {'html': def_html, 'ts': (new Date()).getTime()};
}
}
catch(ee) {
CliqzHumanWeb.contextFromEvent = null;
}
},
captureMouseClickPage: function(ev, contextHTML) {
// if the target is a link of type hash it does not work, it will create a new page without referral
//
var targetURL = CliqzHumanWeb.getURLFromEvent(ev);
if (contextHTML) {
CliqzHumanWeb.contextFromEvent = {
html: contextHTML,
ts: Date.now()
};
} else {
CliqzHumanWeb.contextFromEvent = null;
}
if (targetURL!=null) {
var embURL = CliqzHumanWeb.getEmbeddedURL(targetURL);
if (embURL!=null) targetURL = embURL;
var activeURL = CliqzHumanWeb.currentURL();
if (CliqzHumanWeb.debug) {
_log('captureMouseClickPage>> ' + CliqzHumanWeb.counter + ' ' + targetURL + ' : ' + " active: " + activeURL + " " + (CliqzHumanWeb.state['v'][activeURL]!=null) + " " + ev.target + ' :: ' + ev.target.value + ' >>' + JSON.stringify(CliqzHumanWeb.lastEv));
}
//var activeURL = CliqzHumanWeb.currentURL();
if (CliqzHumanWeb.state['v'][activeURL]!=null) {
CliqzHumanWeb.linkCache[targetURL] = {'s': ''+activeURL, 'time': CliqzHumanWeb.counter};
//Fix same link in 'l'
//Only add if gur. that they are public and the link exists in the double fetch page(Public).it's available on the public page.Such
//check is not done, therefore we do not push the links clicked on that page. - potential record linkage.
//We need to check for redirections and use the final link for 'l' this is why the logic is here. This will
//for sure miss the first time it's see, cause we don't know on mouse click where it redirects.
var linkURL = targetURL;
if(CliqzHumanWeb.httpCache[targetURL]){
if(CliqzHumanWeb.httpCache[targetURL]['status'] == '301'){
linkURL = CliqzHumanWeb.httpCache[targetURL]['location'];
}
}
if(!CliqzHumanWeb.isSuspiciousURL(linkURL) && !CliqzHumanWeb.dropLongURL(linkURL)){
CliqzHumanWeb.getPageFromHashTable(linkURL, function(_res) {
if (_res && (_res['private'] == 0)) {
CliqzHumanWeb.state['v'][activeURL]['c'].push({'l': ''+ CliqzHumanWeb.maskURL(linkURL), 't': CliqzHumanWeb.counter});
}
else if(!_res){
CliqzHumanWeb.state['v'][activeURL]['c'].push({'l': ''+ CliqzHumanWeb.maskURL(linkURL), 't': CliqzHumanWeb.counter});
}
})
}
}
}
if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['mouseclickpage']|0)) > 1 * CliqzHumanWeb.tmult) {
if (CliqzHumanWeb.debug) {
_log('captureMouseClickPage');
}
CliqzHumanWeb.lastEv['mouseclickpage'] = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);;
if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
CliqzHumanWeb.state['v'][activeURL]['e']['md'] += 1;
}
}
},
captureScrollPage: function(ev) {
if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['scrollpage']|0)) > 1 * CliqzHumanWeb.tmult) {
if (CliqzHumanWeb.debug) {
_log('captureScrollPage ');
}
CliqzHumanWeb.lastEv['scrollpage'] = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
CliqzHumanWeb.state['v'][activeURL]['e']['sc'] += 1;
}
}
},
captureCopyPage: function(ev) {
if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['copypage']|0)) > 1 * CliqzHumanWeb.tmult) {
if (CliqzHumanWeb.debug) {
_log('captureCopyPage');
}
CliqzHumanWeb.lastEv['copypage'] = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
CliqzHumanWeb.state['v'][activeURL]['e']['cp'] += 1;
}
}
},
counter: 0,
tmult: 4,
tpace: 250,
lastEv: {},
lastActive: null,
lastActiveAll: null,
getAllOpenPages: function() {
const urls = [];
try {
forEachWindow(win => {
const openTabs = queryActiveTabs(win);
openTabs.forEach(data => {
const url = data.url;
if (url && urls.indexOf(url) === -1 && url.startsWith('about:') === false) {
urls.push(decodeURIComponent(url));
}
});
});
} catch (ee) {
// do nothing, return empty set
}
return urls;
/*
var res = [];
try {
var enumerator = Services.wm.getEnumerator('navigator:browser');
while (enumerator.hasMoreElements()) {
var win = enumerator.getNext();
var gBrowser = win.gBrowser;
if (gBrowser.tabContainer) {
var numTabs = gBrowser.tabContainer.childNodes.length;
for (var i=0; i<numTabs; i++) {
var currentTab = gBrowser.tabContainer.childNodes[i];
var currentBrowser = gBrowser.getBrowserForTab(currentTab);
var currURL = currentBrowser.selectedBrowser.currentURI.spec;
if (currURL.indexOf('about:')!=0) {
res.push(decodeURIComponent(currURL));
}
}
}
}
return res;
}
catch(ee) {
return [];
}
*/
},
init: function(window) {
if(CliqzUtils.getPref("dnt", false)) return;
CliqzUtils.hw = this;
refineFuncMappings = {
"splitF":CliqzHumanWeb.refineSplitFunc,
"parseU":CliqzHumanWeb.refineParseURIFunc,
"maskU":CliqzHumanWeb.refineMaskUrl
};
_log("Init function called:")
var dbConn = CliqzHumanWeb.initDB();
if (CliqzHumanWeb.state == null) {
CliqzHumanWeb.state = {};
}
/*
else {
var util = window.QueryInterface(Components.interfaces.nsIInterfaceRequestor).getInterface(Components.interfaces.nsIDOMWindowUtils);
var win_id = util.outerWindowID;
if (CliqzHumanWeb.windowsMem[win_id] == null) {
CliqzHumanWeb.windowsMem[win_id] = window;
CliqzHumanWeb.windowsRef.push(window);
}
}
var util = window.QueryInterface(Components.interfaces.nsIInterfaceRequestor).getInterface(Components.interfaces.nsIDOMWindowUtils);
var win_id = util.outerWindowID;
if (CliqzHumanWeb.windowsMem[win_id] == null) {
CliqzHumanWeb.windowsMem[win_id] = window;
CliqzHumanWeb.windowsRef.push(window);
}
*/
if (CliqzHumanWeb.pacemakerId==null) {
CliqzHumanWeb.pacemakerId = CliqzUtils.setInterval(CliqzHumanWeb.pacemaker, CliqzHumanWeb.tpace, null);
}
CliqzHumanWeb.fetchAndStoreConfig();
if (CliqzHumanWeb.actionStats==null) CliqzHumanWeb.loadActionStats();
if (CliqzHumanWeb.actionStatsLastSent==null) CliqzHumanWeb.loadActionStatsLastSent();
// Load bloom filter
if(!CliqzHumanWeb.bloomFilter){
CliqzHumanWeb.loadBloomFilter();
}
// Load strict queries
CliqzHumanWeb.loadStrictQueries();
let rsNormal = new ResourceLoader(
["human-web","patterns"],
{
chromeURL: "chrome://cliqz/content/bower_components/patterns/index",
remoteURL : CliqzHumanWeb.patternsURL,
cron: 1 * 20 * 60 * 1000,
}
);
rsNormal.load().then( e => {
CliqzHumanWeb.loadContentExtraction(e, "normal")
});
rsNormal.onUpdate( e => CliqzHumanWeb.loadContentExtraction(e, "normal"));
let rsStrict = new ResourceLoader(
["human-web","patterns-anon"],
{
chromeURL: "chrome://cliqz/content/bower_components/anonpatterns/index",
remoteURL : CliqzHumanWeb.anonPatternsURL,
cron: 1 * 20 * 60 * 1000,
}
);
rsStrict.load().then( e => {
CliqzHumanWeb.loadContentExtraction(e, "strict")
});
rsStrict.onUpdate( e => CliqzHumanWeb.loadContentExtraction(e, "strict"));
// Load quorum config. Update every 20 minutes.
CliqzHumanWeb.safeQuorumProvider = new ResourceLoader(
["hw","quorum-config"],
{
remoteURL: CliqzHumanWeb.SAFE_QUORUM_PROVIDER,
cron: 1 * 20 * 60 * 1000,
updateInterval: 1 * 20 * 60 * 1000,
}
);
CliqzHumanWeb.safeQuorumProvider.load().then( e => {
CliqzHumanWeb.keyExpire = e.expiry * (24 * 60 * 60 * 1000); // Backend sends it in days;
CliqzHumanWeb.oc = e.oc;
CliqzHumanWeb.quorumThreshold = e.threshold;
});
CliqzHumanWeb.safeQuorumProvider.onUpdate(e => {
CliqzHumanWeb.keyExpire = e.expiry * (24 * 60 * 60 * 1000); // Backend sends it in days;
CliqzHumanWeb.oc = e.oc;
CliqzHumanWeb.quorumThreshold = e.threshold;
});
// Load quorum bloom filter
CliqzHumanWeb.loadQuorumBloomFilter();
},
initAtBrowser: function(){
if(CliqzUtils.getPref("dnt", false)) return;
CliqzHumanWeb.activityDistributor.addObserver(CliqzHumanWeb.httpObserver);
},
state: {'v': {}, 'm': [], '_id': Math.floor( Math.random() * 1000 ) },
hashCode: function(s) {
return s.split("").reduce(function(a,b){a=((a<<5)-a)+b.charCodeAt(0);return a&a},0);
},
msgSanitize: function(msg){
//Check and add time , else do not send the message
try {msg.ts = CliqzUtils.getPref('config_ts', null)} catch(ee){};
if(!msg.ts || msg.ts == ''){
return null;
}
// Adding anti-duplicate key, so to detect duplicate messages on the backend.
msg['anti-duplicates'] = Math.floor(Math.random() * 10000000);
if (msg.action == 'page') {
if(msg.payload.tend && msg.payload.tin){
var duration = msg.payload.tend - msg.payload.tin;
_log("Duration spent: " + msg.payload.tend + " : " + msg.payload.tin + " : " + duration );
}
else{
var duration = null;
_log("Duration spent: " + msg.payload.tend + " : " + msg.payload.tin + " : " + duration );
}
msg.payload['dur'] = duration;
delete msg.payload.tend;
delete msg.payload.tin;
// Check for fields which have urls like ref.
// Check if they are suspicious.
// Check if they are marked private.
if(msg.payload.ref){
if(CliqzHumanWeb.isSuspiciousURL(msg.payload['ref'])){
msg.payload['ref'] = null;
}
else{
msg.payload['ref'] = CliqzHumanWeb.maskURL(msg.payload['ref']);
}
// Check if ref. exists in bloom filter, then turn ref to null.
CliqzHumanWeb.getPageFromHashTable(msg.payload.ref, function(_res) {
if (_res) {
if(_res['private'] == 1 ){
msg.payload['ref'] = null;
}
}
})
}
// Remove continuations if not in the same domain for extra safety,
/*
if(msg.payload.c) {
var cleanCont = [];
try {
var mainDom = CliqzHumanWeb.parseURL(msg.payload.url).hostname;
msg.payload.c.forEach(function(e){
if (CliqzHumanWeb.parseURL(e.l).hostname == mainDom) cleanCont.push(e);
});
} catch(ee) {};
msg.payload.c = cleanCont;
}
// Check for title.
if(msg.payload.x.t){
if(CliqzHumanWeb.isSuspiciousTitle(msg.payload.x.t)){
_log("Suspicious Title: " + msg.payload.x.t);
return null;
}
}
else{
_log("Missing Title: " + msg.payload.x.t);
return null;
}
*/
// Remove C
if(msg.payload.c) {
msg.payload.c = null;
}
if (CliqzHumanWeb.dropLongURL(msg.payload.url)==true) {
// the url is not safe, replace by the canonical if exists and is safe
var canonical_url = msg.payload.x.canonical_url;
if (canonical_url != null && canonical_url != '' && CliqzHumanWeb.dropLongURL(canonical_url)==false) {
// the canonical exists and is ok
msg.payload.url = canonical_url;
}
else {
_log("Suspicious url with no/bad canonical: " + msg.payload.url);
return null;
}
}
else {
var canonical_url = msg.payload.x.canonical_url;
if (canonical_url != null && canonical_url != '' && CliqzHumanWeb.dropLongURL(canonical_url)==true) {
// the canonical is not safe, but the url is, remove only the canonical
msg.payload.x.canonical_url = null;
}
}
// validate that is not a shortener url, they are not useful anyway
var short_url = CliqzHumanWeb.isShortenerURL(msg.payload.url);
var short_canonical_url = false;
if (msg.payload.x.canonical_url != null && msg.payload.x.canonical_url != '') {
short_canonical_url = CliqzHumanWeb.isShortenerURL(msg.payload.x.canonical_url);
}
if (short_url || short_canonical_url) return null;
// check if suspiciousURL
if (CliqzHumanWeb.isSuspiciousURL(msg.payload.url)) return null;
if (msg.payload.x.canonical_url != null && msg.payload.x.canonical_url != '') {
if (CliqzHumanWeb.isSuspiciousURL(msg.payload.x.canonical_url)) return null;
}
//Mask the long ugly redirect URLs
if(msg.payload.red){
var cleanRed = [];
msg.payload.red.forEach(function(e){
cleanRed.push(CliqzHumanWeb.maskURL(e));
})
msg.payload.red = cleanRed;
}
// Check for canonical seen or not.
if(msg.payload['x']['canonical_url']) {
if(msg.payload['url'] == msg.payload['x']['canonical_url']){
_log("Canoncial is same: ");
// canonicalSeen = CliqzHumanWeb.canoincalUrlSeen(msg.payload['x']['canonical_url']);
if(msg.payload['csb'] && msg.payload['ft']) {
_log("Canoncial seen before: ");
delete msg.payload.csb;
delete msg.payload.ft;
}
}
// if the url is not replaces by canonical then also clear the csb key.
if(msg.payload['csb']) delete msg.payload.csb;
}
}
// FIXME: this cannot be here, telemetry is only for sending logic. The object needs to be
// handled beforehand!!!
// Canonical URLs and Referrals.
/*
if(CliqzHumanWeb.can_urls[msg.payload.url]){
msg.payload.url = CliqzHumanWeb.can_urls[msg.payload.url];
}
*/
//Check the depth. Just to be extra sure.
if(msg.payload.qr){
if(msg.payload.qr.d > 2){
delete msg.payload.qr;
}
}
// Check if qr.q is suspicious.
if(msg.payload.qr){
if (CliqzHumanWeb.isSuspiciousQuery(msg.payload.qr.q)) {
delete msg.payload.qr;
}
}
//Check for doorway action durl
if(msg.action=='doorwaypage') {
if((CliqzHumanWeb.isSuspiciousURL(msg.payload['durl'])) || (CliqzHumanWeb.isSuspiciousURL(msg.payload['url']))){
return null;
}
if((CliqzHumanWeb.dropLongURL(msg.payload['durl'])) || (CliqzHumanWeb.dropLongURL(msg.payload['url']))){
return null;
}
}
//Remove the msg if the query is too long,
if(msg.action=='query' || msg.action == 'anon-query') {
//Remove the msg if the query is too long,
if ((msg.payload.q == null) || (msg.payload.q == '')) {
return null;
}
else {
if (CliqzHumanWeb.isSuspiciousQuery(msg.payload.q)) {
return null;
}
}
}
return msg;
},
// ****************************
// telemetry, PREFER NOT TO SHARE WITH CliqzUtils for safety, blatant rip-off though
// ****************************
trk: [],
trkTimer: null,
notification: function(payload){
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){};
if(payload && typeof(payload) === 'object'){
payload['ctry'] = location;
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'telemetry', 'payload': payload});
}
else{
_log("Not a valid object, not sent to notification");
}
},
telemetry: function(msg, instantPush) {
if (!CliqzHumanWeb || //might be called after the module gets unloaded
CliqzUtils.getPref('dnt', false) ||
CliqzUtils.isPrivate(CliqzUtils.getWindow())) return;
// Check if host is private or not.
CliqzHumanWeb.isLocalURL(msg).then( status => {
if (!status) {
_log("Check safe quorum");
CliqzHumanWeb.safeQuorumCheck(msg).then( safe => {
_log("Quorum consent ?" + safe);
if (safe) {
msg.ver = CliqzHumanWeb.VERSION;
msg = CliqzHumanWeb.msgSanitize(msg);
_log("Message sanitized");
if (msg) CliqzHumanWeb.incrActionStats(msg.action);
if (msg) CliqzHumanWeb.trk.push(msg);
_log("Added to the queue");
CliqzUtils.clearTimeout(CliqzHumanWeb.trkTimer);
if(instantPush || CliqzHumanWeb.trk.length % 100 == 0){
CliqzHumanWeb.pushTelemetry();
} else {
CliqzHumanWeb.trkTimer = CliqzUtils.setTimeout(CliqzHumanWeb.pushTelemetry, 60000);
}
} else {
// Send telemetry.
CliqzHumanWeb.incrActionStats("droppedQC");
}
});
}
});
},
_telemetry_req: null,
_telemetry_sending: [],
telemetry_MAX_SIZE: 500,
pushTelemetry: function() {
if(CliqzHumanWeb._telemetry_req) return;
// put current data aside in case of failure
CliqzHumanWeb._telemetry_sending = CliqzHumanWeb.trk.splice(0);
var data = JSON.stringify(CliqzHumanWeb._telemetry_sending);
CliqzHumanWeb._telemetry_req = CliqzUtils.promiseHttpHandler('POST', CliqzUtils.SAFE_BROWSING, data, 60000, true);
CliqzHumanWeb._telemetry_req.then( CliqzHumanWeb.pushTelemetryCallback );
CliqzHumanWeb._telemetry_req.catch( CliqzHumanWeb.pushTelemetryError );
},
pushTelemetryCallback: function(req){
try {
var response = JSON.parse(req.response);
CliqzHumanWeb._telemetry_sending = [];
CliqzHumanWeb._telemetry_req = null;
} catch(e){}
},
pushTelemetryError: function(req){
// pushTelemetry failed, put data back in queue to be sent again later
CliqzHumanWeb.trk = CliqzHumanWeb._telemetry_sending.concat(CliqzHumanWeb.trk);
// Remove some old entries if too many are stored, to prevent unbounded growth when problems with network.
var slice_pos = CliqzHumanWeb.trk.length - CliqzHumanWeb.telemetry_MAX_SIZE + 100;
if(slice_pos > 0){
CliqzHumanWeb.trk = CliqzHumanWeb.trk.slice(slice_pos);
}
CliqzHumanWeb._telemetry_sending = [];
CliqzHumanWeb._telemetry_req = null;
},
// ************************ Database ***********************
// source modules/CliqzHistory
// *********************************************************
initDB: function() {
if ( FileUtils.getFile("ProfD", ["cliqz.dbusafe"]).exists() ) {
if (CliqzHumanWeb.olddbConn==null) {
CliqzHumanWeb.olddbConn = Services.storage.openDatabase(FileUtils.getFile("ProfD", ["cliqz.dbusafe"]));
}
CliqzHumanWeb.removeTable();
}
if ( FileUtils.getFile("ProfD", ["cliqz.dbhumanweb"]).exists() ) {
if (CliqzHumanWeb.dbConn==null) {
CliqzHumanWeb.dbConn = Services.storage.openDatabase(FileUtils.getFile("ProfD", ["cliqz.dbhumanweb"]))
}
CliqzHumanWeb.createTable();
}
else {
CliqzHumanWeb.dbConn = Services.storage.openDatabase(FileUtils.getFile("ProfD", ["cliqz.dbhumanweb"]));
CliqzHumanWeb.createTable();
}
return CliqzHumanWeb.dbConn;
},
dbConn: null,
auxSameDomain: function(url1, url2) {
try {
var d1 = CliqzHumanWeb.parseURL(url1).hostname.replace('www.','');
var d2 = CliqzHumanWeb.parseURL(url2).hostname.replace('www.','');
return d1==d2;
} catch(ee) {
return false;
}
},
getPageFromDB: function(url, callback) {
var res = [];
var st = CliqzHumanWeb.dbConn.createStatement("SELECT * FROM usafe WHERE url = :url");
st.params.url = url;
var res = [];
st.executeAsync({
handleResult: function(aResultSet) {
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
res.push({"url": row.getResultByName("url"), "ref": row.getResultByName("ref"), "private": row.getResultByName("private"), "checked": row.getResultByName("checked")});
}
},
handleError: function(aError) {
_log("SQL error: " + aError.message);
callback(true);
},
handleCompletion: function(aReason) {
if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
_log("SQL canceled or aborted");
callback(null);
}
else {
if (res.length == 1) {
callback(res[0]);
}
else {
callback(null);
}
}
}
});
},
getPageFromHashTable: function(url, callback) {
var hash = (md5(url)).substring(0,16);
var r = null;
if(CliqzHumanWeb.bloomFilter){
var sta = CliqzHumanWeb.bloomFilter.testSingle(hash);
if(sta) {
r = {"hash": hash, "private": 1}
}
else{
r = {"hash": hash, "private": 0}
}
}
callback(r);
/*
var res = [];
var st = CliqzHumanWeb.dbConn.createStatement("SELECT * FROM hashusafe WHERE hash = :hash");
st.params.hash = (md5(url)).substring(0,16);
st.executeAsync({
handleResult: function(aResultSet) {
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
res.push({"hash": row.getResultByName("hash"), "private": row.getResultByName("private")});
}
},
handleError: function(aError) {
_log("SQL error: " + aError.message);
callback(true);
},
handleCompletion: function(aReason) {
if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
_log("SQL canceled or aborted");
callback(null);
}
else {
if (res.length == 1) {
callback(res[0]);
}
else {
callback(null);
}
}
}
});
*/
},
getCanUrlFromHashTable: function(canUrl, callback) {
var res = [];
var st = CliqzHumanWeb.dbConn.createStatement("SELECT * FROM hashcans WHERE hash = :hash");
st.params.hash = (md5(canUrl)).substring(0,16);
st.executeAsync({
handleResult: function(aResultSet) {
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
res.push({"hash": row.getResultByName("hash")});
}
},
handleError: function(aError) {
_log("SQL error: " + aError.message);
callback(true);
},
handleCompletion: function(aReason) {
if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
_log("SQL canceled or aborted");
callback(null);
}
else {
if (res.length == 1) {
callback(res[0]);
}
else {
callback(null);
}
}
}
});
},
isPrivate: function(url, depth, callback) {
// returns 1 is private (because of checked, of because the referrer is private)
// returns 0 if public
// returns -1 if not checked yet, handled as public in this cases,
var res = [];
var st = CliqzHumanWeb.dbConn.createStatement("SELECT * FROM usafe WHERE url = :url");
st.params.url = url;
var res = [];
st.executeAsync({
handleResult: function(aResultSet) {
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
res.push({"url": row.getResultByName("url"), "ref": row.getResultByName("ref"), "private": row.getResultByName("private"), "checked": row.getResultByName("checked")});
}
},
handleError: function(aError) {
_log("SQL error: " + aError.message);
callback(true);
},
handleCompletion: function(aReason) {
if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
_log("SQL canceled or aborted");
callback(true);
}
else {
if (res.length == 1) {
if (res[0].ref!='' && res[0].ref!=null) {
// the urls already exists in the DB, it has been seen before
if (depth < 10) {
if (CliqzHumanWeb.auxSameDomain(res[0].ref, url)) {
CliqzHumanWeb.isPrivate(res[0].ref, depth+1, function(priv) {
callback(priv);
});
}
else callback(false);
}
else {
// set to private (becasue we are not sure so beter safe than sorry),
// there is a loop of length > 10 between a <- b <- .... <- a, so if we do not
// break recursion it will continue to do the SELECT forever
//
callback(true);
}
}
else {
callback(false);
}
}
else {
callback(true);
}
}
}
});
},
parseHostname: function(hostname) {
var o = {'hostname': null, 'username': '', 'password': '', 'port': null};
var h = hostname;
var v = hostname.split('@');
if (v.length > 1) {
var w = v[0].split(':');
o['username'] = w[0];
o['password'] = w[1];
h = v[1];
}
v = h.split(':');
if (v.length > 1) {
o['hostname'] = v[0];
o['port'] = parseInt(v[1]);
}
else {
o['hostname'] = v[0];
o['port'] = 80;
}
return o;
},
parseURL: function(url) {
// username, password, port, path, query_string, hostname, protocol
var o = {};
var v = url.split('://');
if (v.length >= 2) {
o['protocol'] = v[0];
var s = v.slice(1, v.length).join('://');
v = s.split('/');
// Check for hostname, if not present then return null.
if (v[0] === '') return null;
// Check if the hostname is invalid by checking for special characters.
// Only special characters like - and _ are allowed.
var hostnameRegex = /[?!@#\$\^\&*\)\(+=]/g;
if (hostnameRegex.test(v[0])) return null;
var oh = CliqzHumanWeb.parseHostname(v[0]);
o['hostname'] = oh['hostname'];
o['port'] = oh['port'];
o['username'] = oh['username'];
o['password'] = oh['password'];
o['path'] = '/';
o['query_string'] = null;
if (v.length>1) {
s = v.splice(1, v.length).join('/');
v = s.split('?')
o['path'] = '/' + v[0];
if (v.length>1) {
o['query_string'] = v.splice(1, v.length).join('?');
}
v = o['path'].split(';');
o['path'] = v[0];
if (v.length>1) o['query_string'] = v.splice(1, v.length).join(';') + '&' + (o['query_string'] || '');
v = o['path'].split('#');
o['path'] = v[0];
if (v.length>1) o['query_string'] = v.splice(1, v.length).join('#') + '&' + (o['query_string'] || '');
}
}
else {
return null;
}
return o;
},
addURLtoDB: function(url, ref, paylobj) {
var tt = new Date().getTime();
var se = CliqzHumanWeb.checkSearchURL(url);
if (se > -1 ){
return
}
//Check if url is in hashtable
var ft = 1;
var privateHash = false;
CliqzHumanWeb.getPageFromHashTable(url, function(_res) {
if (_res) {
if(_res['private'] == 1 ){
privateHash = true;
}
else{
ft = 0;
}
}
else{
// we never seen it, let's add it
paylobj['ft'] = true;
}
})
// Need to add if canonical is seen before or not.
// This is helpful, becuase now we replace the url with canonical incase of dropLongUrl(url) => true.
// Hence, in the event log, lot of URL's look ft => true.
if(paylobj['x'] && paylobj['x']['canonical_url'] && paylobj['x']['canonical_url'] != url){
CliqzHumanWeb.getCanUrlFromHashTable(paylobj['x']['canonical_url'], function(_res) {
if (_res) {
paylobj['csb'] = true;
}
})
}
var stmt = CliqzHumanWeb.dbConn.createStatement("SELECT url, checked, ft, private, payload FROM usafe WHERE url = :url");
stmt.params.url = url;
var res = [];
stmt.executeAsync({
handleResult: function(aResultSet) {
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
res.push({'url': row.getResultByName("url"), 'checked': row.getResultByName("checked"), 'ft' :row.getResultByName('ft'), 'private' :row.getResultByName('private'), 'payload' :row.getResultByName('payload') });
}
},
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
_log("SQL canceled or aborted");
}
else {
if (res.length == 0 && !privateHash ){
var setPrivate = false;
var st = CliqzHumanWeb.dbConn.createStatement("INSERT INTO usafe (url,ref,last_visit,first_visit, reason, private, checked,payload, ft) VALUES (:url, :ref, :last_visit, :first_visit, :reason, :private, :checked, :payload, :ft)");
st.params.url = url;
st.params.ref = ref;
st.params.last_visit = tt;
st.params.first_visit = tt;
st.params.ft = ft;
st.params.payload = JSON.stringify(paylobj || {});
if (paylobj['x']==null) {
// page data structure is empty, so no need to double fetch, is private
st.params.checked = 1;
st.params.private = 1;
st.params.reason = 'empty page data';
setPrivate = true;
_log("Setting private because empty page data");
}
else if (CliqzHumanWeb.isSuspiciousURL(url)) {
// if the url looks private already add it already as checked and private
st.params.checked = 1;
st.params.private = 1;
st.params.reason = 'susp. url';
setPrivate = true;
_log("Setting private because suspiciousURL");
}
else {
if (CliqzHumanWeb.httpCache401[url]) {
st.params.checked = 1;
st.params.private = 1;
st.params.reason = '401';
setPrivate = true;
_log("Setting private because of 401");
}
else {
st.params.checked = 0;
st.params.private = 0;
st.params.reason = '';
setPrivate = false;
}
}
//while (st.executeStep()) {};
st.executeAsync({
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if(CliqzHumanWeb.debug){
_log("Insertion success add urltoDB");
}
}
});
if(setPrivate){
CliqzHumanWeb.setAsPrivate(url);
}
}
else if(res.length > 0){
if (res[0]['checked']==0) {
//Need to aggregate the engagement metrics.
var metricsBefore = JSON.parse(res[0]['payload'])['e'];
var metricsAfter = paylobj['e'];
paylobj['e'] = CliqzHumanWeb.aggregateMetrics(metricsBefore, metricsAfter);
//Since not checked it is still the ft.
if(res[0]['ft']==1){
paylobj['ft'] = true;
}
var st = CliqzHumanWeb.dbConn.createStatement("UPDATE usafe SET last_visit = :last_visit, payload = :payload WHERE url = :url");
st.params.url = url;
st.params.last_visit = tt;
st.params.payload = JSON.stringify(paylobj || {});
//while (st.executeStep()) {};
st.executeAsync({
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if(CliqzHumanWeb.debug){
_log("Insertion success");
}
}
});
paylobj['e'] = {'cp': 0, 'mm': 0, 'kp': 0, 'sc': 0, 'md': 0};
}
else{
if (res[0]['checked']==1 && res[0]['private'] == 0) {
//Need to aggregate the engagement metrics.
var metricsBefore = res[0]['payload']['e'];
var metricsAfter = paylobj['e'];
paylobj['e'] = CliqzHumanWeb.aggregateMetrics(metricsBefore, metricsAfter);
var st = CliqzHumanWeb.dbConn.createStatement("UPDATE usafe SET last_visit = :last_visit, payload = :payload, checked = :checked WHERE url = :url");
st.params.url = url;
st.params.last_visit = tt;
st.params.payload = JSON.stringify(paylobj || {});
st.params.checked = 0;
//while (st.executeStep()) {};
st.executeAsync({
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if(CliqzHumanWeb.debug){
_log("Insertion success");
}
}
});
paylobj['e'] = {'cp': 0, 'mm': 0, 'kp': 0, 'sc': 0, 'md': 0};
}
}
}
}
}
});
},
setAsPrivate: function(url) {
if(CliqzHumanWeb.bloomFilter){
CliqzHumanWeb.bloomFilter.addSingle((md5(url)).substring(0,16));
}
var st = CliqzHumanWeb.dbConn.createStatement("DELETE from usafe WHERE url = :url");
st.params.url = url;
//while (st.executeStep()) {};
st.executeAsync({
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if(CliqzHumanWeb.debug){
_log("Delete success");
}
}
});
if(CliqzHumanWeb.state['v'][url]){
delete CliqzHumanWeb.state['v'][url];
}
CliqzHumanWeb.dumpBloomFilter();
// incr stats for private page, to see public vs private pages marked.
// mp : when double fetch failed or url was marked private.
CliqzHumanWeb.incrActionStats("mp");
/*
//Add has in the hashusafe table
var hash_st = CliqzHumanWeb.dbConn.createStatement("INSERT OR IGNORE INTO hashusafe (hash, private) VALUES (:hash, :private)")
hash_st.params.hash = (md5(url)).substring(0,16);
hash_st.params.private = 1;
//while (hash_st.executeStep()) {};
hash_st.executeAsync({
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if(CliqzHumanWeb.debug){
_log("Insertion success");
}
}
});
if (CliqzHumanWeb.debug) {
_log('MD5: ' + url + md5(url) + " ::: " + (md5(url)).substring(0,16));
}
*/
},
setAsPublic: function(url) {
var st = CliqzHumanWeb.dbConn.createStatement("DELETE from usafe WHERE url = :url")
st.params.url = url;
//while (st.executeStep()) {};
st.executeAsync({
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if(CliqzHumanWeb.debug){
_log("Insertion success");
}
}
});
if(CliqzHumanWeb.state['v'][url]){
delete CliqzHumanWeb.state['v'][url];
}
/*
//Add has in the hashusafe table
var hash_st = CliqzHumanWeb.dbConn.createStatement("INSERT OR IGNORE INTO hashusafe (hash, private) VALUES (:hash, :private)")
hash_st.params.hash = (md5(url)).substring(0,16);
hash_st.params.private = 0;
//while (hash_st.executeStep()) {};
hash_st.executeAsync({
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if(CliqzHumanWeb.debug){
_log("Insertion success");
}
}
});
if (CliqzHumanWeb.debug) {
_log('MD5: ' + url + md5(url));
}
*/
},
listOfUnchecked: function(cap, sec_old, fixed_url, callback) {
var tt = new Date().getTime();
var stmt = null;
if (fixed_url == null) {
// all urls
stmt = CliqzHumanWeb.dbConn.createAsyncStatement("SELECT url, payload FROM usafe WHERE last_visit < :last_visit and private = :private and checked = :checked LIMIT :cap;");
}
else {
stmt = CliqzHumanWeb.dbConn.createAsyncStatement("SELECT url, payload FROM usafe WHERE last_visit < :last_visit and url = :url and private = :private and checked = :checked LIMIT :cap;");
stmt.params.url = fixed_url;
}
stmt.params.last_visit = (tt - sec_old*1000);
stmt.params.private = 0;
stmt.params.cap = cap;
stmt.params.checked = 0;
var res = [];
stmt.executeAsync({
handleResult: function(aResultSet) {
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
res.push([row.getResultByName("url"), JSON.parse(row.getResultByName("payload")) ]);
}
},
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
_log("SQL canceled or aborted");
}
else {
if(res.length > 0){
_log("Got the result: " + res[0]);
}
callback(res.splice(0,cap), null);
}
}
});
},
processUnchecks: function(listOfUncheckedUrls) {
var url_pagedocPair = {};
for(var i=0;i<listOfUncheckedUrls.length;i++) {
var url = listOfUncheckedUrls[i][0];
var page_doc = listOfUncheckedUrls[i][1];
var page_struct_before = page_doc['x'];
url_pagedocPair[url] = page_doc;
CliqzHumanWeb.isPrivate(url, 0,function(isPrivate) {
if (isPrivate) {
var st = CliqzHumanWeb.dbConn.createStatement("UPDATE usafe SET reason = :reason, checked = :checked, private = :private , ft = :ft WHERE url = :url");
st.params.url = url;
st.params.checked = 1;
st.params.private = 1;
st.params.ft = 0;
st.params.reason = 'priv. st.';
//while (st.executeStep()) {};
st.executeAsync({
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if(CliqzHumanWeb.debug){
_log("Insertion success private");
}
}
});
_log("Marking as private via is private");
CliqzHumanWeb.setAsPrivate(url);
}
else {
_log("Going for double fetch: " + url);
// only do doubleFetch for the same url 3 times in a row
// (set up as CliqzHumanWeb.MAX_NUMBER_DOUBLEFETCH_ATTEMPS).
// If more attemps are tried then the url is marked as private.
// Prevent infinite loop if the doubleFetch causes the browser
// to crash (issue #2213)
//
CliqzHumanWeb.loadRecord('last-double-fetch', function(data) {
var obj = null;
if (data==null) obj = {'url': url, 'count': 1};
else {
obj = JSON.parse(data);
if (obj.url!=url) obj = {'url': url, 'count': 1};
else {
try {
obj['count'] += 1;
} catch(err) {
obj['count'] = 1;
}
}
}
CliqzHumanWeb.saveRecord('last-double-fetch', JSON.stringify(obj));
if (obj.count > CliqzHumanWeb.MAX_NUMBER_DOUBLEFETCH_ATTEMPS) {
CliqzHumanWeb.setAsPrivate(url);
}
else {
CliqzHumanWeb.doubleFetch(url, url_pagedocPair[url]);
}
});
}
});
}
},
// to invoke in console: CliqzHumanWeb.listOfUnchecked(1000000000000, 0, null, function(x) {console.log(x)})
forceDoubleFetch: function(url) {
CliqzHumanWeb.listOfUnchecked(1000000000000, 0, url, CliqzHumanWeb.processUnchecks);
},
outOfABTest: function() {
(CliqzHumanWeb.dbConn.executeSimpleSQLAsync || CliqzHumanWeb.dbConn.executeSimpleSQL)('DROP TABLE usafe;');
},
removeTable: function(reason) {
try{
(CliqzHumanWeb.olddbConn.executeSimpleSQLAsync || CliqzHumanWeb.olddbConn.executeSimpleSQL)('DROP TABLE usafe;');
}catch(ee){};
},
debugInterface: function() {
var ww = Components.classes["@mozilla.org/embedcomp/window-watcher;1"]
.getService(Components.interfaces.nsIWindowWatcher);
try{var win = ww.openWindow(null, "chrome://cliqzmodules/content/debugInterface",
"debugInterface", null, null);}catch(ee){_log(ee)}
},
loadContentExtraction: function(resp, mode){
//Load content extraction.
if (mode === "normal"){
var patternConfig = resp;
CliqzHumanWeb.searchEngines = patternConfig["searchEngines"];
CliqzHumanWeb.extractRules = patternConfig["scrape"];
CliqzHumanWeb.payloads = patternConfig["payloads"];
CliqzHumanWeb.idMappings = patternConfig["idMapping"];
CliqzHumanWeb.rArray = [];
patternConfig["urlPatterns"].forEach(function(e){
CliqzHumanWeb.rArray.push(new RegExp(e));
})
}
else if(mode === "strict"){
var patternConfig = resp;
CliqzHumanWeb.anonSearchEngines = patternConfig["searchEngines"];
CliqzHumanWeb.anonExtractRules = patternConfig["scrape"];
CliqzHumanWeb.anonPayloads = patternConfig["payloads"];
CliqzHumanWeb.anonIdMappings = patternConfig["idMapping"];
CliqzHumanWeb.anonRArray = [];
patternConfig["urlPatterns"].forEach(function (e) {
CliqzHumanWeb.anonRArray.push(new RegExp(e));
});
}
},
checkForEmail: function(str) {
if (str.match(/[a-z0-9\-_@]+(@|%40|%(25)+40)[a-z0-9\-_]+\.[a-z0-9\-_]/i) != null) return true;
else return false;
},
checkForLongNumber: function(str, max_number_length) {
var cstr = str.replace(/[^A-Za-z0-9]/g,'');
var lcn = 0;
var maxlcn = 0;
var maxlcnpos = null;
for(let i=0;i<cstr.length;i++) {
if (cstr[i]>='0' && cstr[i]<='9') lcn+=1;
else {
if (lcn > maxlcn) {
maxlcn = lcn;
maxlcnpos = i;
lcn = 0;
}
else lcn = 0;
}
}
if (lcn > maxlcn) {
maxlcn = lcn;
maxlcnpos = cstr.length;
lcn = 0;
}
else lcn = 0;
if (maxlcnpos!=null && maxlcn > max_number_length) return cstr.slice(maxlcnpos-maxlcn,maxlcnpos);
else return null;
},
fetchAndStoreConfig: function(){
//Load latest config.
CliqzUtils.httpGet(CliqzHumanWeb.configURL,
function success(req){
if(!CliqzHumanWeb) return;
try {
var config = JSON.parse(req.response);
for(var k in config){
CliqzUtils.setPref('config_' + k, config[k]);
}
} catch(e){};
},
function error(res){
_log('Error loading config. ')
}, 5000);
},
checkURL: function(cd, url, ruleset){
var pageContent = cd;
//var rArray = new Array(new RegExp(/\.google\..*?[#?&;]q=[^$&]+/), new RegExp(/.search.yahoo\..*?[#?&;]p=[^$&]+/), new RegExp(/.linkedin.*?\/pub\/dir+/),new RegExp(/\.bing\..*?[#?&;]q=[^$&]+/),new RegExp(/.*/))
//scrap(4, pageContent)
let rArray = [];
let searchEngines = [];
if (ruleset === "normal"){
rArray = CliqzHumanWeb.rArray;
searchEngines = CliqzHumanWeb.searchEngines;
}
else if (ruleset === "strict"){
rArray = CliqzHumanWeb.anonRArray;
searchEngines = CliqzHumanWeb.anonSearchEngines
}
for(var i=0;i<rArray.length;i++){
if (rArray[i].test(url)){
CliqzHumanWeb.extractContent(i, pageContent, url, ruleset);
//Do not want to continue after search engines...
if(searchEngines.indexOf(''+i) != -1 ){return;}
if (CliqzHumanWeb.debug) {
_log('Continue further after search engines ');
}
}
}
},
checkSearchURL: function(url){
var idx = null;
var reref = /\.google\..*?\/(?:url|aclk)\?/;
for(var i=0;i<CliqzHumanWeb.rArray.length;i++){
if(CliqzHumanWeb.rArray[i].test(url)){
//Do not want to continue after search engines... && !reref.test(url)
if(CliqzHumanWeb.searchEngines.indexOf(''+i) != -1 ){
idx = i;
return idx;
}
else{
if (CliqzHumanWeb.debug) {
_log('Not search engine ' + i + CliqzHumanWeb.searchEngines);
}
return -1;
}
}
}
},
checkAnonSearchURL: function (url) {
var idx = null;
for (var i = 0; i < CliqzHumanWeb.anonRArray.length; i++) {
if (CliqzHumanWeb.anonRArray[i].test(url)) {
//Do not want to continue after search engines... && !reref.test(url)
if (CliqzHumanWeb.anonSearchEngines.indexOf('' + i) != -1) {
;
idx = i;
return idx;
} else {
if (CliqzHumanWeb.debug) {
_log('Not search engine ' + i + CliqzHumanWeb.searchEngines);
}
return -1;
}
}
}
},
extractContent: function(ind, cd, url, ruleset){
var scrapeResults = {};
var eventMsg = {};
var rules = {};
var key = "";
var rule = "";
var payloadRules = [];
if(ruleset === "normal"){
rules = CliqzHumanWeb.extractRules[ind];
payloadRules = CliqzHumanWeb.payloads[ind];
}
else if(ruleset === "strict"){
rules = CliqzHumanWeb.anonExtractRules[ind];
payloadRules = CliqzHumanWeb.anonPayloads[ind];
}
if (CliqzHumanWeb.debug) {
_log('rules' + rules + ind);
}
var urlArray = [];
var titleArray = [];
for(key in rules){
var _keys = Object.keys(rules[key]);
if (CliqzHumanWeb.debug) {
_log('keys' + _keys);
}
var innerDict = {};
_keys.forEach(function(each_key){
if(rules[key][each_key]['type'] == 'standard'){
//Depending on etype, currently only supporting url. Maybe ctry too.
if(rules[key][each_key]['etype'] == 'url'){
var qurl = url;
var functionsApplied = (rules[key][each_key]['functionsApplied'] || null);
// Check if the value needs to be refined or not.
if(functionsApplied){
qurl = functionsApplied.reduce(function(attribVal, e){
if(refineFuncMappings.hasOwnProperty(e[0])){
return refineFuncMappings[e[0]](attribVal,e[1],e[2]);
}
else{
return attribVal;
}
},qurl)
}
innerDict[each_key] = [qurl];
}
if(rules[key][each_key]['etype'] == 'ctry'){
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){};
innerDict[each_key] = [location];
}
}
else if(rules[key][each_key]['type'] == 'searchQuery'){
urlArray = CliqzHumanWeb._getAttribute(cd,key,rules[key][each_key]['item'], rules[key][each_key]['etype'], rules[key][each_key]['keyName'],(rules[key][each_key]['functionsApplied'] || null))
innerDict[each_key] = urlArray;
CliqzHumanWeb.searchCache[ind] = {'q' : urlArray[0], 't' : CliqzHumanWeb.idMappings[ind]};
}
else{
urlArray = CliqzHumanWeb._getAttribute(cd,key,rules[key][each_key]['item'], rules[key][each_key]['etype'], rules[key][each_key]['keyName'],(rules[key][each_key]['functionsApplied'] || null))
innerDict[each_key] = urlArray;
}
})
if(CliqzHumanWeb.messageTemplate[ind]){
CliqzHumanWeb.messageTemplate[ind][key] = innerDict;
}
else{
CliqzHumanWeb.messageTemplate[ind] = {};
CliqzHumanWeb.messageTemplate[ind][key] = innerDict;
}
//Check if array has values.
var _mergeArr = CliqzHumanWeb.mergeArr(CliqzHumanWeb.messageTemplate[ind][key]);
if(_mergeArr.length > 0){
scrapeResults[key] = _mergeArr;
}
}
for(rule in payloadRules){
CliqzHumanWeb.createPayload(scrapeResults, ind, rule, ruleset);
}
},
mergeArr: function(arrS){
var messageList = [];
var allKeys = [];
allKeys = Object.keys(arrS);
arrS[allKeys[0]].forEach(function(e,idx){var innerDict ={};messageList.push(allKeys.map(function(e,_idx,arr){innerDict[e]=arrS[e][idx];return innerDict})[0])})
return messageList;
},
_getAttribute: function(cd,parentItem,item,attrib,keyName,functionsApplied){
var arr = [];
var rootElement = Array.prototype.slice.call(cd.querySelectorAll(parentItem));
for(var i=0;i<rootElement.length;i++){
var val = rootElement[i].querySelector(item);
if (val){
//Not Null
var innerDict = {};
var attribVal = val[attrib] || val.getAttribute(attrib);
// Check if the value needs to be refined or not.
if(functionsApplied){
attribVal = functionsApplied.reduce(function(attribVal, e){
if(refineFuncMappings.hasOwnProperty(e[0])){
return refineFuncMappings[e[0]](attribVal,e[1],e[2]);
}
else{
return attribVal;
}
},attribVal)
}
arr.push(innerDict[keyName] = attribVal);
}
else{
var innerDict = {}
arr.push(innerDict[keyName] = val);
}
}
return arr;
},
createPayload: function(scrapeResults, idx, key, payloadRule){
try{
if(payloadRule === "normal"){
var payloadRules = CliqzHumanWeb.payloads[idx][key];
}else if(payloadRule === "strict"){
var payloadRules = CliqzHumanWeb.anonPayloads[idx][key];
}
if (payloadRules['type'] == 'single' && payloadRules['results'] == 'single' ){
scrapeResults[key].forEach(function(e){
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){};
e['ctry'] = location;
CliqzHumanWeb.sendMessage(payloadRules, e)
})
}
else if (payloadRules['type'] == 'single' && payloadRules['results'] == 'custom' ){
var payload = {};
payloadRules['fields'].forEach(function(e){
try{payload[e[1]] = scrapeResults[e[0]][0][e[1]]}catch(ee){};
CliqzHumanWeb.sendMessage(payloadRules, payload)
})
}
else if (payloadRules['type'] == 'query' && payloadRules['results'] == 'clustered'){
var payload = {};
payloadRules['fields'].forEach(function(e){
if (e.length > 2){
var joinArr = {};
for(var i=0;i<scrapeResults[e[0]].length;i++){
joinArr['' + i] = scrapeResults[e[0]][i];
}
payload[e[1]] = joinArr;
}
else{
payload[e[1]] = scrapeResults[e[0]][0][e[1]];
}
})
CliqzHumanWeb.sendMessage(payloadRules, payload);
}
else if (payloadRules['type'] == 'query' && payloadRules['results'] == 'scattered'){
var payload = {};
payloadRules['fields'].forEach(function(e){
if (e.length > 2){
var joinArr = {};
var counter = 0;
e[0].forEach(function(eachPattern){
for(var i=0;i<scrapeResults[eachPattern].length;i++){
joinArr['' + counter] = scrapeResults[eachPattern][i];
counter += 1;
}
})
if(Object.keys(joinArr).length > 0){
payload[e[1]] = joinArr;
}
}
else{
payload[e[1]] = scrapeResults[e[0]][0][e[1]];
}
})
CliqzHumanWeb.sendMessage(payloadRules, payload)
}
}
catch(ee){}
},
sendMessage: function(payloadRules, payload){
if (CliqzHumanWeb.debug) {
_log("sendMessage" );
}
var c = true;
var e = "";
var allKeys = Object.keys(payload);
for(e in payloadRules['fields']){
if (allKeys.indexOf(payloadRules['fields'][e][1]) == -1){
c = false;
}
else{
allKeys.forEach(function(each_field){
if (!(payload[each_field])){
c = false;
}
})
}
}
if(c){
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': payloadRules['action'], 'payload':payload})
}
CliqzHumanWeb.messageTemplate = {};
},
refineSplitFunc: function(splitString, splitON, arrPos){
var result = splitString.split(splitON)[arrPos];
if(result){
return decodeURIComponent(result);
}
else{
return decodeURIComponent(splitString);
}
},
refineParseURIFunc: function(url, extractType, keyName){
var urlParts = CliqzHumanWeb.parseURL(url);
if(urlParts && urlParts.query_string) {
var result = CliqzHumanWeb.parseQueryString(urlParts.query_string);
if(extractType == 'qs'){
if(result[keyName]){
return decodeURIComponent(result[keyName][0]);
}
else{
return url;
}
}
} else {
return url;
}
},
refineReplaceFunc: function(replaceString, replaceWhat, replaceWith ){
var result = decodeURIComponent(replaceString.replace("",replaceWhat,replaceWith));
return result;
},
refineMaskUrl: function(url){
var result = CliqzHumanWeb.maskURL(url);
return result;
},
getTabID: function(){
try {
var win = CliqzUtils.getWindow();
var gBrowser = win.gBrowser;
return win.__SSi + ":" + gBrowser.mCurrentTab._tPos;
}
catch(e) {
return null;
}
},
createTable: function(){
var usafe = "create table if not exists usafe(\
url VARCHAR(255) PRIMARY KEY NOT NULL,\
ref VARCHAR(255),\
last_visit INTEGER,\
first_visit INTEGER,\
reason VARCHAR(256), \
private BOOLEAN DEFAULT 0,\
checked BOOLEAN DEFAULT 0, \
payload VARCHAR(4096), \
ft BOOLEAN DEFAULT 1 \
)";
var hash_usafe = "create table if not exists hashusafe(\
hash VARCHAR(32) PRIMARY KEY NOT NULL,\
private BOOLEAN DEFAULT 0 \
)";
var hash_cans = "create table if not exists hashcans(\
hash VARCHAR(32) PRIMARY KEY NOT NULL \
)";
var telemetry = "create table if not exists telemetry(\
id VARCHAR(24) PRIMARY KEY NOT NULL,\
data VARCHAR(1000000) \
)";
(CliqzHumanWeb.dbConn.executeSimpleSQLAsync || CliqzHumanWeb.dbConn.executeSimpleSQL)(usafe);
(CliqzHumanWeb.dbConn.executeSimpleSQLAsync || CliqzHumanWeb.dbConn.executeSimpleSQL)(hash_usafe);
(CliqzHumanWeb.dbConn.executeSimpleSQLAsync || CliqzHumanWeb.dbConn.executeSimpleSQL)(hash_cans);
(CliqzHumanWeb.dbConn.executeSimpleSQLAsync || CliqzHumanWeb.dbConn.executeSimpleSQL)(telemetry);
},
aggregateMetrics:function (metricsBefore, metricsAfter){
var aggregates = {"cp": 0,"mm": 0,"kp": 0,"sc": 0,"md": 0};
if (CliqzHumanWeb.debug) {
_log("aggregates: " + JSON.stringify(metricsBefore) + JSON.stringify(metricsAfter));
}
var _keys = Object.keys(aggregates);
for(var i=0;i<_keys.length;i++){
aggregates[_keys[i]] = metricsBefore[_keys[i]] + metricsAfter[_keys[i]];
}
if (CliqzHumanWeb.debug) {
_log("aggregates: " + JSON.stringify(aggregates));
}
return aggregates;
},
isSuspiciousTitle: function(title) {
// 1. Need to check if the title is suspicious or not.
// 2. Title should should not contain number greater than 8.
// 3. Title should not contain html.
var vt = title.split(' ');
for(var i=0;i<vt.length;i++) {
if (vt[i].length>CliqzHumanWeb.rel_segment_len) {
var cstr = vt[i].replace(/[^A-Za-z0-9]/g,'');
if (cstr.length > CliqzHumanWeb.rel_segment_len) {
if (CliqzHumanWeb.isHash(cstr)) return true;
}
}
}
if (CliqzHumanWeb.checkForLongNumber(title, 8) != null) {
return true;
}
if (CliqzHumanWeb.checkForEmail(title)) {
return true;
}
if(/<[^<]+>/.test(title)){
return true;
}
return false;
},
auxProbString: function(h, p, s, c) {
// h[i] = k : there are k bins (letter) with i balls (repetitions) on it,
// h[2] = 1 : there is one letter that is repeated twice
if (h[1] == s) {
// recursion stop condition
return 1.0;
}
else {
var tot_p = 0.0;
var kkp = 1.0;
for(let i=h.length-1; i>1; i--) {
if (h[i]!=0) {
// it has repetitions,
var pi = (h[i-1]+1)*p;
var h2 = h.slice();
h2[i] -= 1;
if (false && (h2[i]==0 && h2[i-1]==0 && i>1)) {
// minimize number of recursions
h2[i-1] += 1;
kkp = kkp * p;
}
else {
h2[i-1] += 1;
var lab = h2.join('-');
if (!c[lab]) c[lab] = CliqzHumanWeb.auxProbString(h2, p, s, c);
tot_p += (kkp * pi * c[lab]);
}
}
}
return tot_p;
}
},
probString: function(str) {
var bins = {}
for(let i=0;i<str.length;i++) bins[str[i]] = (bins[str[i]] || 0) + 1
var keys = Object.keys(bins);
var p = 1.0 / keys.length;
var k = str.length - keys.length;
var h = []
for(let i=0;i<str.length+1;i++) h[i] = 0;
var s = 0;
var max_freq = 0;
for(let i=0;i<keys.length+1;i++) {
var freq = bins[keys[i]];
if (freq > 0) {
h[bins[keys[i]]] += 1;
s += 1;
if (freq > max_freq) {
max_freq = freq;
}
}
}
h = h.slice(0, max_freq+1);
var cache = {};
var prob_conf = CliqzHumanWeb.auxProbString(h, p, s, cache);
// probability of the string to be random, e.g. hash. Better to keep it for
// strings > 15, lower probability means not random, > 0.25, random.
return prob_conf;
},
isHashProb: function(str) {
var log_prob = 0.0;
var trans_c = 0;
str = str.replace(/[^A-Za-z0-9]/g,'');
for(var i=0;i<str.length-1;i++) {
var pos1 = CliqzHumanWeb.probHashChars[str[i]]
var pos2 = CliqzHumanWeb.probHashChars[str[i+1]];
if (pos1 && pos2) {
log_prob += CliqzHumanWeb.probHashLogM[pos1][pos2];
trans_c += 1;
}
}
if (trans_c > 0) return Math.exp(log_prob/trans_c);
else return Math.exp(log_prob);
},
isHash: function(str) {
var p = CliqzHumanWeb.isHashProb(str);
return (p < CliqzHumanWeb.probHashThreshold);
},
parseQueryString: function (q){
/* parse the query */
var x = q.replace(/;/g, '&').split('&'), i, name, t;
/* q changes from string version of query to object */
for (q={}, i=0; i<x.length; i++){
t = x[i].split('=', 2);
name = unescape(t[0]);
if (!q[name]) q[name] = [];
if (t.length > 1){
q[name][q[name].length] = unescape(t[1]);
}
/* next two lines are nonstandard */
else q[name][q[name].length] = true;
}
return q;
},
escapeSQL: function(str) {
return str.replace(/[\0\x08\x09\x1a\n\r"'\\\%]/g, function(char) {
switch (char) {
case "'":
return "''";
default:
return char;
/*case "\0":
return "\\0";
case "\x08":
return "\\b";
case "\x09":
return "\\t";
case "\x1a":
return "\\z";
case "\n":
return "\\n";
case "\r":
return "\\r";
case "\"":
case "'":
case "\\":
case "%":
return "\\"+char; */
}
});
},
checkActiveUsage: function(){
//This function needs to be scheduled every one hour.
var oldUsage = 0;
try {oldUsage = CliqzUtils.getPref('config_activeUsageCount', 0)} catch(ee){};
var activeUsage = CliqzHumanWeb.activeUsage + oldUsage;
if(activeUsage && activeUsage > CliqzHumanWeb.activeUsageThreshold){
//Sample event to be sent
var payload = {};
payload['status'] = true;
payload['t'] = CliqzHumanWeb.getTime();
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){};
payload['ctry'] = location;
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'alive', 'payload':payload})
CliqzHumanWeb.activeUsage = 0;
CliqzUtils.setPref('config_activeUsage', new Date().getTime().toString());
CliqzUtils.setPref('config_activeUsageCount', 0);
}
},
insertCanUrl: function(canUrl){
//Add canUrl in the hashcans table
var hash_st = CliqzHumanWeb.dbConn.createStatement("INSERT OR IGNORE INTO hashcans (hash) VALUES (:hash)")
hash_st.params.hash = (md5(canUrl)).substring(0,16);
//while (hash_st.executeStep()) {};
hash_st.executeAsync({
handleError: function(aError) {
_log("SQL error: " + aError.message);
},
handleCompletion: function(aReason) {
if(CliqzHumanWeb.debug){
_log("Insertion success canonical URL");
}
}
});
if (CliqzHumanWeb.debug) {
_log('MD5: ' + canUrl + md5(canUrl));
}
},
getPageStatusCount: function(callback) {
var st = CliqzHumanWeb.dbConn.createStatement("SELECT count(1) as 'total',SUM(CASE private when 0 then 1 else 0 end) AS 'allowed',SUM(CASE private when 1 then 1 else 0 end) AS 'notallowed' FROM hashusafe;");
var res = [];
st.executeAsync({
handleResult: function(aResultSet) {
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
res.push({"total": row.getResultByName("total"), "allowed": row.getResultByName("allowed"), "not-allowed": row.getResultByName("notallowed")});
}
},
handleError: function(aError) {
_log("SQL error: " + aError.message);
callback(true);
},
handleCompletion: function(aReason) {
if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
_log("SQL canceled or aborted");
callback(null);
}
else {
if (res.length == 1) {
callback(res);
}
else {
callback(null);
}
}
}
});
},
saveRecord: function(id, data) {
if(!(CliqzHumanWeb.dbConn)) return;
var st = CliqzHumanWeb.dbConn.createStatement("INSERT OR REPLACE INTO telemetry (id,data) VALUES (:id, :data)");
st.params.id = id;
st.params.data = data;
st.executeAsync({
handleError: function(aError) {
if(CliqzHumanWeb && CliqzHumanWeb.debug){
_log("SQL error: " + aError.message);
}
},
handleCompletion: function(aReason) {
if(CliqzHumanWeb && CliqzHumanWeb.debug){
_log("Insertion success save record");
}
}
});
},
loadRecord: function(id, callback) {
var stmt = CliqzHumanWeb.dbConn.createAsyncStatement("SELECT id, data FROM telemetry WHERE id = :id;");
stmt.params.id = id;
var fres = null;
var res = [];
stmt.executeAsync({
handleResult: function(aResultSet) {
if(!(CliqzHumanWeb)) return;
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
if (row.getResultByName("id")==id) {
res.push(row.getResultByName("data"));
}
else {
_log("There are more than one record");
callback(null);
}
break;
}
},
handleError: function(aError) {
if(!(CliqzHumanWeb)) return;
_log("SQL error: " + aError.message);
callback(null);
},
handleCompletion: function(aReason) {
if(!(CliqzHumanWeb)) return;
if (res.length == 1) callback(res[0]);
else callback(null);
}
});
},
loadActionStats: function() {
CliqzHumanWeb.loadRecord('actionStats', function(data) {
if (data==null) {
_log("There was no data on action stats");
CliqzHumanWeb.actionStats = {};
}
else {
try {
CliqzHumanWeb.actionStats = JSON.parse(data);
} catch(ee) {
CliqzHumanWeb.actionStats = {};
}
}
});
},
loadActionStatsLastSent: function() {
CliqzHumanWeb.loadRecord('actionStats_last_send', function(data) {
if (data==null) {
_log("There was no data on CliqzHumanWeb.actionstats");
CliqzHumanWeb.actionStatsLastSent = CliqzHumanWeb.getTime().slice(0,8);
CliqzHumanWeb.saveActionStatsLastSent();
}
else CliqzHumanWeb.actionStatsLastSent = data;
});
},
incrActionStats: function(action) {
if(!CliqzHumanWeb.actionStats) return;
if (CliqzHumanWeb.actionStats[action] == null) CliqzHumanWeb.actionStats[action] = 0;
if (CliqzHumanWeb.actionStats['total'] == null) CliqzHumanWeb.actionStats['total'] = 0;
CliqzHumanWeb.actionStats[action]++;
CliqzHumanWeb.actionStats['total']++;
},
saveActionStats: function() {
if (CliqzHumanWeb.actionStats && Object.keys(CliqzHumanWeb.actionStats).length > 0) {
CliqzHumanWeb.saveRecord('actionStats', JSON.stringify(CliqzHumanWeb.actionStats));
}
},
sendActionStats: function() {
var payl;
if (CliqzHumanWeb.actionStats && Object.keys(CliqzHumanWeb.actionStats).length > 0) {
payl = {'data': CliqzHumanWeb.actionStats};
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'hw.telemetry.actionstats', 'payload': payl});
// reset the state
CliqzHumanWeb.actionStats = {};
CliqzHumanWeb.saveActionStats();
}
},
saveActionStatsLastSent: function() {
CliqzHumanWeb.saveRecord('actionStats_last_send', CliqzHumanWeb.actionStatsLastSent);
},
saveStrictQueries: function() {
_log("Saving local table");
CliqzHumanWeb.saveRecord('localStrictQueries', JSON.stringify(CliqzHumanWeb.strictQueries));
},
sendActionStatsIfNeeded: function() {
// Send action stats once per day.
// Day resolution.
var timestamp = CliqzHumanWeb.getTime().slice(0,8);
if (timestamp != CliqzHumanWeb.actionStatsLastSent) {
// it's not the same timestamp (hour) of the last time that was sent
// or the first install (defaults to current timestamp)
CliqzHumanWeb.actionStatsLastSent = timestamp;
CliqzHumanWeb.saveActionStatsLastSent();
CliqzHumanWeb.sendActionStats();
}
},
dumpBloomFilter: function(){
var bf = [].slice.call(CliqzHumanWeb.bloomFilter.buckets);
if(bf){
CliqzHumanWeb.saveRecord('bf', bf.join("|"));
}
},
loadBloomFilter: function(){
CliqzHumanWeb.loadRecord('bf', function(data) {
if (data==null) {
_log("There was no data on CliqzHumanWeb.bf");
CliqzHumanWeb.bloomFilter = new CliqzBloomFilter.BloomFilter(Array(bloomFilterSize).join('0'),bloomFilterNHashes);
}
else {
var _data = data.split("|").map(Number);
CliqzHumanWeb.bloomFilter = new CliqzBloomFilter.BloomFilter(_data,bloomFilterNHashes);
}
});
},
loadStrictQueries: function(){
CliqzHumanWeb.loadRecord('localStrictQueries', function(data) {
if (data==null || data.length == 0) {
_log("There was no data on CliqzHumanWeb.bf");
CliqzHumanWeb.strictQueries = [];
}
else {
CliqzHumanWeb.strictQueries = JSON.parse(data);
}
});
},
getDBConn: function(){
return CliqzHumanWeb.dbConn || CliqzHumanWeb.initDB();
},
auxGetQuery: function(){
CliqzHumanWeb.strictQueries.forEach( function(e, idx) {
var t = Date.now();
if((t - e.ts) > (e.tDiff * 60 * 1000)) {
CliqzHumanWeb.auxGetPageData(e.qurl, null, e.qurl,function(url, page_data, ourl, x){
let cd = CliqzHumanWeb.docCache[url]['doc'];
CliqzHumanWeb.checkURL(cd, url, "strict");
}, function(a,b,c,d){
_log("Error aux>>>> " + d)
});
CliqzHumanWeb.strictQueries.splice(idx, 1);
CliqzHumanWeb.saveStrictQueries();
}
})
},
isSuspiciousQuery: function(query) {
//Remove the msg if the query is too long,
if (query.length > 50) return true;
if (query.split(' ').length > 7) return true;
// Remove the msg if the query contains a number longer than 7 digits
// can be 666666 but also things like (090)90-2, 5555 3235
// note that full dates will be removed 2014/12/12
//
var haslongnumber = CliqzHumanWeb.checkForLongNumber(query, 7);
if (haslongnumber!=null) return true;
//Remove if email (exact), even if not totally well formed
if (CliqzHumanWeb.checkForEmail(query)) return true;
//Remove if query looks like an http pass
if (/[^:]+:[^@]+@/.test(query)) return true;
//Remove if email
if (/^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/.test(query)) return true;
var v = query.split(' ');
for(let i=0;i<v.length;i++) {
if (v[i].length > 20) return true;
if (/[^:]+:[^@]+@/.test(v[i])) return true;
if (/^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/.test(v[i])) return true;
}
if (query.length > 12) {
var cquery = query.replace(/[^A-Za-z0-9]/g,'');
if (cquery.length > 12) {
var pp = CliqzHumanWeb.isHashProb(cquery);
// we are a bit more strict here because the query
// can have parts well formed
if (pp < CliqzHumanWeb.probHashThreshold*1.5) return true;
}
}
return false;
},
isLocalURL: function(msg){
// We need to check for action page, if the URLs in the message
// are not private because of local domains. like fritzbox or admin.example.com.
let promise = new Promise(function(resolve, reject){
if (msg.action == 'page') {
// Get all the urls in the payload.
let urls = [];
urls.push(msg.payload.url);
if (msg.payload.ref) urls.push(msg.payload.ref);
if (msg.payload.red) {
msg.payload.red.forEach( redURL => {
urls.push(redURL);
})
}
_log("All urls in the message:" + JSON.stringify(urls));
// Check for each URL if the host is public or private,
// If any of the host is private then it should resolve as true and exit.
// Else should resolve as not private.
Promise.all(urls.map(CliqzHumanWeb.isHostNamePrivate)).then(results => {
_log(JSON.stringify(results));
// Now that we have checked all the URLS, if any of the URL resulted as private
// We drop the message.
if (results.indexOf(true) > -1) {
_log("Contains private URL");
resolve(true);
} else {
_log("URLs are public");
resolve(false);
}
});
} else {
resolve(false);
}
});
return promise;
},
isHostNamePrivate: function(url){
let promise = new Promise(function(resolve, reject){
var host = null;
try {
host = CliqzHumanWeb.parseURL(url).hostname;
} catch(ee) {
resolve(true);
};
// If the parsing of the host fails for some reason,
// we would mark it as private.
if(!host || host === '') {
resolve(true);
}
dnsService.asyncResolve(host,
0,
{
onLookupComplete: function(request, record, status) {
if (!Components.isSuccessCode(status)) {
// Handle error here
resolve(false);
}
let address = record.getNextAddrAsString();
_log("Host= " + host + " Address: " + address);
if (CliqzHumanWeb.isIPInternal(address)) {
_log("Host= " + host + " Address: " + address + "private");
resolve(true);
return;
} else {
_log("Host= " + host + " Address: " + address + "public");
resolve(false);
}
}
},
null
);
});
return promise;
},
isIPInternal: function(ip) {
// Need to check for ipv6.
const ipSplit = ip.split(".");
if (parseInt(ipSplit[0]) === 10 ||
(parseInt(ipSplit[0]) === 172 && (parseInt(ipSplit[1]) >= 16 && parseInt(ipSplit[1]) <= 31)) ||
(parseInt(ipSplit[0]) === 192 && parseInt(ipSplit[1]) === 168) ||
(parseInt(ipSplit[0]) === 127) ||
(parseInt(ipSplit[0]) === 0)) {
return true;
} else {
return false;
}
},
validFrameCount: function(struct_bef, struct_aft) {
//
// To take into account, the transition state, when the extension is updated
// Data saved in the DB will not have the key nifsh, hence we should return true
// for those cases.
//
if (struct_bef.nifsh == null || struct_aft.nifsh == null || struct_bef.nifsh !=struct_aft.nifsh) {
_log("fovalidDoubleFetch: number of internal iframes does not match");
return false;
}
return true;
},
validFrameSetCount: function(struct_bef, struct_aft) {
//
// To take into account, the transition state, when the extension is updated
// Data saved in the DB will not have the key nfsh, hence we should return true
// for those cases.
//
if (struct_bef.nfsh ==null || struct_aft.nfsh==null || struct_bef.nfsh!=struct_aft.nfsh) {
_log("fovalidDoubleFetch: number of internal frameset does not match");
return false;
}
return true;
},
performQC: function(msg){
if (msg.action === "page") {
let parse_url = CliqzHumanWeb.parseURL(msg.payload.url);
if (msg.payload.qr && (msg.payload.qr.t === "cl" || msg.payload.qr.t === "othr")) {
if (parse_url && parse_url.path.length > 1) return true;
} else if (!msg.payload.qr) {
if (parse_url && parse_url.path.length > 1) return true;
}
}
return false;
},
safeQuorumCheck: function(msg) {
//
// Check for conditions.
//
let promise = new Promise( (resolve, reject) => {
if (CliqzHumanWeb.performQC(msg)) {
_log("Perform QC: true");
let url = msg.payload.url;
CliqzHumanWeb.sha1(url).then( hashedUrl => {
CliqzHumanWeb.sendQuorumIncrement(hashedUrl).then( status => {
if (status) {
CliqzHumanWeb.getQuorumConsent(hashedUrl, e => {
let _result = JSON.parse(e.response).result;
_log("Quorum consent: " + _result);
resolve(_result);
});
}
});
});
} else {
_log("Perform QC: false");
resolve(true);
}
});
return promise;
},
sha1: function(s) {
// Pass the message to the web-worker for SHA-1.
let promise = new Promise( (resolve, reject) => {
let wCrypto = new Worker('chrome://cliqz/content/hpn/crypto-worker.js');
wCrypto.onmessage = function(e){
let result = e.data.result;
wCrypto.terminate();
resolve(result);
};
wCrypto.postMessage({
"msg":s,
"type":"hw-sha1"
});
});
return promise;
},
sendQuorumIncrement: function(hashedUrl){
let promise = new Promise( (resolve, reject) => {
// Check for hashed URL in quorum bloom filter;
CliqzHumanWeb.isPageVisitedQuorumBloomFilter(hashedUrl).then( status => {
if (status) {
resolve(true);
} else {
let payload = "?hu=" + hashedUrl + "&oc=" + CliqzHumanWeb.oc;
let _rp = CliqzHumanWeb.SAFE_QUORUM_ENDPOINT + "incrquorum";
CliqzHumanWeb.sendInstantMessage(_rp, payload, e => {
CliqzHumanWeb.setPageVisitQuorumBloomFilter(hashedUrl);
resolve(true);
});
}
});
});
return promise;
},
getQuorumConsent: function(hashedUrl, callback){
let payload = "?hu=" + hashedUrl;
let _rp = CliqzHumanWeb.SAFE_QUORUM_ENDPOINT + "checkquorum";
CliqzHumanWeb.sendInstantMessage(_rp, payload, callback);
},
sendInstantMessage: function(_rp, payload, callback){
const uid = Math.floor(Math.random() * 10000000);
CliqzSecureMessage.queriesID[uid] = callback;
CliqzSecureMessage.wCrypto.postMessage({
msg: { action: 'instant',
type: 'cliqz',
ts: '',
ver: '1.5',
payload: payload,
rp: _rp,
},
uid: uid,
type: 'instant',
sourcemap: CliqzSecureMessage.sourceMap,
upk: CliqzSecureMessage.uPK,
dspk: CliqzSecureMessage.dsPK,
sspk: CliqzSecureMessage.secureLogger,
queryproxyip: CliqzSecureMessage.queryProxyIP,
});
},
registerQuorumBloomFilters: function(){
let promise = new Promise( (resolve, reject) => {
// Check for current date.
let currentDay = CliqzHumanWeb.getTime().slice(0,8);
// Check if quorumBF exists for current date.
if (Object.keys(CliqzHumanWeb.quorumBloomFilters).indexOf(currentDay) === -1) {
_log("Need to create quorum bloom filter for: " + currentDay);
let bloomFilterSize = 10000; // User is unlikely to visit more than 10000 URLs in day. Size is approx. 12 KB. per Bloom filter.
let bloomFilter = new CliqzBloomFilter.BloomFilter(Array(bloomFilterSize).join('0'),bloomFilterNHashes);
CliqzHumanWeb.quorumBloomFilters[currentDay] = bloomFilter;
}
let bf = CliqzHumanWeb.quorumBloomFilters[currentDay];
resolve(bf);
});
return promise;
},
setPageVisitQuorumBloomFilter: function(hashedUrl) {
CliqzHumanWeb.registerQuorumBloomFilters().then( bf => {
if (bf) {
bf.addSingle(hashedUrl);
CliqzHumanWeb.dumpQuorumBloomFilter();
return true;
}
});
},
isPageVisitedQuorumBloomFilter: function(hashedUrl) {
let promise = new Promise( (resolve, reject) => {
Object.keys(CliqzHumanWeb.quorumBloomFilters).forEach( eachDay => {
var status = CliqzHumanWeb.quorumBloomFilters[eachDay].testSingle(hashedUrl);
if (status) {
resolve(true);
}
});
resolve(false);
});
return promise;
},
dumpQuorumBloomFilter: function(){
let quorumBF = {};
Object.keys(CliqzHumanWeb.quorumBloomFilters).forEach( eachDay => {
let _bf = [].slice.call(CliqzHumanWeb.quorumBloomFilters[eachDay].buckets);
quorumBF[eachDay] = _bf.join("|");
});
CliqzHumanWeb.saveRecord('quorumbf', JSON.stringify(quorumBF));
},
loadQuorumBloomFilter: function(){
CliqzHumanWeb.loadRecord('quorumbf', function(data) {
if (data) {
let jData = JSON.parse(data);
_log("Loading quorum bloom filter");
Object.keys(jData).forEach( eachDay => {
let _data = jData[eachDay].split("|").map(Number);
let bloomFilter = new CliqzBloomFilter.BloomFilter(_data,bloomFilterNHashes);
CliqzHumanWeb.quorumBloomFilters[eachDay] = bloomFilter;
});
}
});
},
expireQuorumBloomFilter: function(){
// Need to pass the string as YYYY, MM-1, DD to convert to date object.
let currentDay = CliqzHumanWeb.getTime().slice(0,8);
let dateToday = new Date(currentDay.slice(0,4), currentDay.slice(4,6)-1, currentDay.slice(6,8));
Object.keys(CliqzHumanWeb.quorumBloomFilters).forEach( eachDay => {
let registerDate = new Date(eachDay.slice(0,4), eachDay.slice(4,6)-1, eachDay.slice(6,8));
let diff = (dateToday - registerDate);
if (diff > CliqzHumanWeb.keyExpire) {
delete CliqzHumanWeb.quorumBloomFilters[eachDay];
}
});
CliqzHumanWeb.dumpQuorumBloomFilter();
},
};
export default CliqzHumanWeb;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment