Skip to content

Instantly share code, notes, and snippets.

@konark-cliqz konark-cliqz/human-web.es Secret
Last active Nov 30, 2016

Embed
What would you like to do?
human-web code base
import CliqzEvents from "core/events";
import CliqzBloomFilter from "human-web/bloom-filter";
import { utils } from "core/cliqz";
import md5 from "core/helpers/md5";
import ResourceLoader from 'core/resource-loader';
import { queryActiveTabs } from 'core/tabs';
import { forEachWindow } from 'platform/browser';
import CliqzSecureMessage from 'hpn/main';
Components.utils.import("resource://gre/modules/Services.jsm");
Components.utils.import("resource://gre/modules/FileUtils.jsm");
Cu.import('resource://gre/modules/XPCOMUtils.jsm');
const dnsService = Components.classes["@mozilla.org/network/dns-service;1"]
.createInstance(Components.interfaces.nsIDNSService);
var nsIAO = Components.interfaces.nsIHttpActivityObserver;
var nsIHttpChannel = Components.interfaces.nsIHttpChannel;
var refineFuncMappings ;
var gadurl = /\.google..*?\/(aclk)\?/;
/*
Configuration for Bloomfilter
*/
var bloomFilterSize = 500001;
var falsePositive = 0.01;
var bloomFilterNHashes = 7;
function _log(msg){
try{
if(CliqzHumanWeb.debug) {
CliqzUtils.log(msg, CliqzHumanWeb.LOG_KEY);
}
}
catch(e){CliqzUtils.log(e, CliqzHumanWeb.LOG_KEY)};
}
function getRandomIntInclusive(min, max) {
min = Math.ceil(min);
max = Math.floor(max);
return Math.floor(Math.random() * (max - min + 1)) + min;
}
function getHTML(...args) {
return utils.callAction('core', 'getHTML', args);
}
var CliqzHumanWeb = {
VERSION: '2.4',
WAIT_TIME: 2000,
LOG_KEY: 'humanweb',
debug: false,
httpCache: {},
httpCache401: {},
queryCache: {},
privateCache: {},
UrlsCache :false,
docCache: {},
strictMode: false,
qs_len:30,
rel_part_len:18,
rel_segment_len:15,
doubleFetchTimeInSec: 3600,
can_urls: {},
deadFiveMts: 5,
deadTwentyMts: 20,
msgType:'humanweb',
probHashLogM: [[-1.839225984234144, -1.8009413231413045, -2.5864601561900273, -2.779077348461893, -2.41154163187108, -2.8701669802592216, -3.010853183897427, -2.9831997811803244, -3.0258844950086354, -3.028549665472028, -4.922879838696041, -4.503195676571833, -4.598154161855354, -5.027344698225922, -5.217723368153992, -5.366375752342653, -5.195909278900248, -3.2940850310881387, -5.181468249477294, -6.110371416838184, -5.766202229535159, -5.393352786265537, -4.920024681004049, -6.049294408676102, -6.399015490326156, -4.690163484884073, -7.459851804835359, -5.825337077308845, -4.675000760327133, -5.3635239851672845, -6.331713706782754, -6.169284763337202, -5.951151191317628, -5.768078242705815, -7.332129653597336, -6.766704624275413, -6.01222435569866, -5.268617743244878, -6.279416117827666, -5.861972559492891, -5.838071468059843, -6.267250173459698, -6.283981033030117, -6.454759516784266, -6.5195694451700765, -6.83526729571271, -6.458157413359671, -6.359696067797872, -6.250241538141395, -6.647172527007435, -6.973987597161365, -6.4598607011160425, -7.685547265665763, -6.5912026847359835, -5.794718807999361, -6.474629988987015, -7.135098972554303, -6.880960523371609, -6.747429130747086, -7.733118660641065, -8.398636456057481, -7.6323973914922005], [-2.11332750230783, -2.3060786022344897, -2.3740965556720726, -2.5398307619465608, -2.463417892707307, -2.130953109899374, -2.8009573227909073, -2.9401508827086436, -2.748847204575824, -2.8414237914951976, -5.010884149435608, -4.7716590964777925, -4.868359687249472, -5.322380370933768, -5.037006822424893, -5.279762905036231, -5.635270908532894, -2.9219817650555973, -5.50179037155463, -6.5406369920023, -6.311742070591071, -5.516975693542294, -5.7251712834206305, -6.083331008626942, -6.409455776722731, -4.856277028595425, -7.356357334609669, -6.255977591877104, -5.042089425488359, -5.723688703460408, -6.9306288690141455, -5.879866127104103, -6.269949621534028, -6.969512838475713, -7.475373123729266, -6.802062729887527, -6.566109586768108, -6.661695002244713, -6.634070998648396, -6.128265317507457, -6.0654489711795785, -6.79728322915323, -6.99559673771516, -6.996654379273295, -7.671972036661277, -7.840255377779173, -7.132455920432357, -7.233113706563278, -6.737551430374816, -7.2706736012479585, -7.430290364348156, -6.991903776514698, -8.559572276072503, -7.231105001502197, -6.117005677632925, -6.669675284003577, -7.87022016648111, -7.635233211015519, -7.18343028641663, -8.53463332772525, -8.567176875457722, -7.573077285525099], [-1.6313545940843748, -2.457209014039589, -2.565380346827546, -2.767788786784649, -2.6386222432933284, -2.595661570703591, -2.6893427324813977, -2.7198733792574403, -2.720533202987637, -2.8467380805237736, -5.029918409868431, -4.178710463028381, -5.07588590483658, -5.057436988529895, -4.899938336310833, -5.271744381251988, -5.715419865537325, -3.1739071041721116, -5.367102243680773, -6.224018730172038, -5.866973141306898, -5.730015933050199, -5.360958031920617, -6.20201967617667, -6.328452890969734, -4.837569916970598, -7.520833339090353, -5.8325529174204185, -4.950768880962046, -5.339171801246006, -6.500538943869661, -6.141626179853517, -5.826929659866057, -6.371549418153106, -7.095602530626063, -6.863928261303842, -6.26532998038965, -6.486382994639529, -6.568804701240378, -5.928030195562504, -6.138501621140467, -6.375855129864318, -6.672567424416514, -6.87690014131182, -7.500698430681298, -7.486404619424152, -7.110483757622713, -6.948864783187929, -6.7689909475117735, -7.44680608044408, -7.305175362028806, -6.490004000406482, -8.278142463867583, -7.100043692626031, -6.262757631704339, -6.785644569035485, -7.917859578552653, -6.966883288690608, -6.7663306639529575, -7.827911341889713, -7.709275282211631, -7.650980633730635], [-2.2483023852308777, -2.2601431921520803, -2.543211818297523, -2.6290975996017654, -2.597012293697696, -2.425038774903284, -2.6464962853598264, -2.7268820909874645, -2.7170988469992086, -2.6384412943652213, -4.8059842634060495, -4.801920790322523, -4.881995682865955, -4.399130557719554, -4.977525552072196, -5.254333597613871, -5.4861286120783905, -3.172333855510997, -5.311009967578493, -6.3787775808955045, -6.060833750357407, -5.179488625881995, -5.284849141539821, -6.185226477248653, -6.342526765978808, -5.0477193482508715, -7.229508117457252, -5.92913163950562, -4.781563610047119, -5.175693021409531, -6.686598785853124, -5.953265540842049, -5.765108625614716, -6.249636653394711, -7.053405713688212, -6.871084156894258, -4.746700678954438, -4.435206614193364, -6.2492260600816145, -5.437566316202071, -5.90294982336378, -6.5708663077474965, -6.4774110336687505, -6.642451113295277, -7.490560332753359, -7.638809897851904, -7.037026821604173, -6.792453878834064, -6.557926605402683, -7.183535297458448, -7.556615154408894, -6.675849393697125, -7.929241490870054, -6.920447880012594, -5.937211886970303, -6.536908132533395, -7.266281062975223, -7.138583909596181, -6.9407325511841, -7.611188320541975, -8.063402408224212, -7.7539194100649205], [-2.0350349956205984, -2.348602646771946, -2.41594411405956, -2.651370395735765, -2.563410442434685, -2.706962611141542, -2.7917112923152776, -2.7483543933766477, -2.6785026736567357, -2.8111298756441037, -4.759837736546805, -4.20618015484438, -4.787453629928326, -4.659538126267765, -5.015532806050122, -5.056335259588051, -5.339468177159593, -3.040667184111712, -5.203144447478613, -6.2264472230490675, -5.7097357671058155, -5.585194192596145, -5.035187712987706, -5.542988699592872, -6.206189420830567, -4.864552817623237, -7.387161020524848, -5.130725109803191, -4.710070938057912, -5.015532806050122, -5.563665939332496, -5.8412571088023135, -5.744609136156329, -6.206575148897066, -6.842735398083401, -6.794361813799185, -5.850672107736916, -6.088449236501511, -6.348519254409916, -5.725588459198469, -6.064735388153122, -6.479956671206651, -6.2209527037314265, -6.70094736915976, -7.3405268040104845, -7.140183173146506, -6.467358872953416, -6.7233339772218645, -6.184068000080206, -6.381516598393398, -7.406185056529545, -6.368731036096427, -7.402351186018824, -6.6091509166536255, -5.9623588984476195, -6.352529061520146, -7.133337180638517, -7.24783614109371, -6.901266731308386, -7.234806640803376, -7.432126848507545, -7.494477663675224], [-2.0215812860615885, -2.455637406643175, -2.4330253990141046, -2.6113902739466517, -2.6033787533338284, -2.6337646574813993, -2.7111759743895725, -2.7471499108067454, -2.721672070749707, -2.7644985821403028, -4.771735346669196, -4.967298931497775, -4.806933772233218, -4.796223062219612, -4.702049659123834, -4.615986584200591, -5.422222957790021, -3.024083897225106, -5.341473161524181, -5.691932137893965, -5.870247680316981, -5.6516120254419615, -4.85258639108815, -5.9684706438872475, -6.205102036008844, -4.762283950640148, -7.106211026775939, -5.754565801069223, -4.28552037163959, -5.083154204506742, -6.514533306380853, -5.91282033178054, -5.801594696670316, -6.211408552601299, -6.884100000075919, -6.5060032575028, -6.051550266183317, -6.1528452495165125, -6.237040028392324, -5.845664774991169, -5.83750622157883, -6.376088248894053, -6.3936457603588925, -6.58430154569624, -7.071724850704769, -7.213982427018076, -6.56730052905341, -6.608473779316545, -6.403307671270629, -6.973970456584425, -7.485700648480843, -6.5060032575028, -7.126013654072119, -6.592294210440276, -5.525669597265563, -6.0726189713973024, -6.168884085714272, -7.057863242618145, -6.702467385782517, -7.84259108644045, -7.932742183434748, -6.889071012797939], [-2.3515175517946965, -2.4895377785815813, -2.455794452354539, -2.6177065611128625, -2.3916765698655924, -2.55893036105207, -2.621684157393488, -2.6988517931658524, -2.630805311213068, -2.6634082117951667, -4.639123440452939, -4.108116635400416, -4.850674635561831, -4.879642999369406, -5.020506071184806, -5.079702479761731, -5.3856932140647995, -2.922465393995544, -5.272571266829349, -6.095451326503016, -5.703507123198923, -5.5861660464475555, -5.144197286007818, -5.722031888813277, -6.28586346876822, -4.872555039237073, -6.886991750607913, -5.521389523166842, -4.712785087276782, -4.712557943607434, -6.258849400356963, -5.781080159458192, -5.683518571573409, -6.497978247391754, -6.801813554699944, -6.640350228026399, -6.23670827447975, -6.480528140170159, -6.503409353749256, -6.035279500709713, -6.187868456644626, -6.634900623258835, -6.227883924459252, -6.665648019797556, -7.5457712876636505, -7.422426740346121, -6.235666064901307, -6.769319721223498, -6.343344078699495, -6.975730165249003, -7.353206363454935, -6.635677322327007, -7.937813375439674, -6.762240045635436, -6.147006740966892, -6.741296871790193, -7.715813334827543, -7.183705495090042, -6.938232467377743, -7.593211012735211, -8.021194984378726, -7.326496213127361], [-2.404503840183417, -2.43761707422273, -2.4108252303609095, -2.2999425986283435, -2.542843475674806, -2.6029429346724786, -2.584141922741018, -2.5855127818195496, -2.5510561981552526, -2.647681869737238, -4.782820501953572, -4.808208666935694, -4.979563332219715, -5.008211525808261, -5.046432738628459, -5.142457459370653, -5.485491564024772, -3.1144173778919617, -5.417825855041213, -6.245547233803318, -5.938268453446564, -5.963941744764968, -5.450843208258146, -6.059373833665894, -6.223182084343739, -4.942745700375246, -7.208409268623897, -6.038143491827719, -4.6277437642171355, -4.892513996642899, -6.694766741586792, -6.117821568685912, -5.949820065666706, -6.502648045547855, -6.835922059321975, -6.694766741586792, -6.361312919572388, -6.403294392961231, -6.118769885494771, -5.676056523505988, -6.406451957946552, -6.5244722133881785, -6.6333977952105005, -6.6574953467895615, -7.573786078663716, -7.446115322000925, -6.883692335590661, -6.903250344490318, -6.3498910941692, -6.905331510694142, -7.588115904218541, -6.879623153983935, -8.035692497676832, -6.939233062369824, -5.959077527634608, -6.324513515108117, -7.549688527084656, -7.120692921566297, -6.988939594552065, -7.891731808755324, -7.727259481120012, -7.430143376434873], [-2.2924692182301323, -2.450539739657926, -2.3769703926915984, -2.532604294663703, -2.55393918066343, -2.609718771364289, -2.6246425013179655, -2.661744915258029, -2.6011009513028984, -2.6236780353341946, -4.669477044233787, -4.1521285006409165, -4.955491059584871, -4.818890308190988, -5.030853379357302, -5.002039294312803, -5.278250342600015, -3.076829272811893, -5.307731406455737, -5.826171280206974, -6.022604681193609, -5.8364861502494865, -5.222160875948972, -5.86734719439116, -6.263453577434854, -5.1217643044016885, -7.32073988901422, -5.758783848671988, -4.666476541899311, -5.259001445170042, -6.374002568031498, -6.083417735921755, -5.695962812152638, -6.490715185357981, -6.417147740477192, -6.55681384079998, -5.431378631989581, -5.843302320137157, -6.369727752361356, -5.638321273173884, -6.50456582229188, -6.275027142828877, -6.210627212225794, -6.774279201200845, -7.54625217416398, -7.377527245335997, -6.968174323388823, -6.961544464850153, -6.5738576027632725, -6.808653231033201, -7.247759185607984, -6.667823395181603, -8.06567857870257, -6.862038055489511, -5.923075085846576, -6.772454383176216, -7.732534132174032, -6.980444415980637, -6.935456028765855, -7.926350068393722, -7.409760739910981, -7.357691884279851], [-2.395140177772492, -2.3972905877762423, -2.4380054501613637, -2.59514809923619, -2.5304957080424493, -2.553494441315067, -2.592658670737186, -2.6234135177213647, -2.5068257242437872, -2.493912396989218, -4.415128005729072, -4.8162395242315545, -4.805681457561184, -4.959588437581088, -5.013259465745109, -5.070148436368585, -5.778302005668334, -3.0289910877331776, -5.527143864017425, -6.11105402601701, -6.123887370951674, -5.975929983744964, -5.442482986164537, -6.151079378646925, -6.120416006196215, -5.065655600056955, -7.071974876907828, -5.957253719340739, -4.781773843113505, -5.354948164056644, -6.520450343119245, -6.293473674919533, -5.817016517849025, -6.85914069525248, -6.81356318675616, -6.777584585720929, -5.97764525003172, -6.226877237430227, -6.02961643394924, -5.9729353291005545, -6.418715298488868, -5.532354351604477, -6.803710890313149, -6.889653320113874, -7.354541848692839, -7.502757793000282, -6.938924369120657, -6.214291571987144, -6.749334119435077, -7.237514956739779, -7.394443065445024, -6.457482478746571, -7.947443614261728, -6.86121538479334, -5.698583920623846, -6.845759126556648, -7.683896617545187, -7.180188461548061, -6.757725777071325, -7.8704825731256, -7.881944016644606, -7.107203568791572], [-6.7847523397462, -4.956382101946025, -5.55168468117927, -6.54694976738503, -6.679027997805541, -6.920449307304025, -6.651631877885628, -7.028532297007316, -6.819873006155468, -6.9668156836069475, -5.07824502580267, -3.6978375281608122, -3.2158001218369128, -3.221520357616586, -3.5888259593254497, -4.18118917224718, -3.0226572253435795, -3.971027081545084, -3.562365839815367, -6.241860395850777, -3.738063429007306, -2.286143322088719, -2.8154088123469325, -1.9366972011829824, -6.402763696295053, -4.09673956284193, -6.7632686126933175, -2.217315348390325, -2.7898081670861736, -2.2970370063901697, -2.7943042308650328, -4.45525252224197, -5.6170558845712195, -5.634999368405547, -4.204524838877218, -5.247540598697068, -7.679731693977602, -7.496500850432476, -7.694782629326121, -7.660949127015419, -8.444625153562612, -8.02242837580043, -7.9107342568656245, -7.844229700045118, -8.496568431723452, -8.90528981102539, -7.744710798608082, -7.796294560812227, -7.4084137546993, -8.54315037261458, -8.96995880381818, -7.702981796781846, -9.818649422090688, -7.982756374523151, -7.087136791691802, -7.726484291615801, -9.302472289566692, -8.264363329731667, -8.520253749464962, -9.573314897467998, -10.041792973404897, -9.03352354731742], [-5.530908158078372, -5.12660572425173, -5.285205739999349, -5.724471900365845, -5.873312125055024, -6.16534879225732, -6.148934708170209, -6.3610443530793574, -6.311033932504697, -6.085846837289822, -1.9318371570117603, -3.9897678769234868, -4.264721024425275, -4.508665668126993, -1.397989588361542, -5.165043688633805, -5.1354703161064155, -4.710076673495516, -2.2812710196631696, -5.744808587095395, -6.153180999051661, -2.7741841902487896, -4.878049696777151, -5.0409257316634655, -2.3898944653854226, -5.13595610674021, -7.841822106027579, -2.8471437524600627, -3.4340095911141377, -4.619316513108799, -2.3549200090866647, -5.921518406898233, -5.302848234969303, -6.786138406296854, -4.207750888408638, -6.039012800612939, -7.841822106027579, -7.704962923310382, -7.065002449743519, -7.589296290560376, -8.260096172428325, -8.035112694192108, -7.871344545293901, -7.140827013169332, -8.545938317958099, -8.836893715754849, -7.993349965830133, -7.27788665694764, -7.741565502287929, -7.888977243944857, -8.111485672976682, -7.875096894912451, -10.091297373707675, -7.392485113590403, -6.863268166852988, -7.610229500226842, -8.901213681542407, -8.92241588919301, -8.313038591698524, -9.803615301255896, -9.310181420201774, -9.00812940746704], [-5.262771475279199, -4.784242541566078, -5.27970172852834, -5.684918512820975, -5.945292208575153, -6.218979090786592, -6.228233983716081, -6.354509212158639, -6.204315840281487, -6.442281485719593, -2.526983467749325, -5.114296740462473, -4.530006859098648, -5.422625873895182, -2.9651924820539826, -5.615416770210166, -5.759915004501759, -0.738654569892248, -3.9951800832331146, -7.496424744983864, -2.271649654828947, -3.6688863968618732, -4.479075949205008, -6.406971077861438, -2.7704548687771466, -5.064362048924221, -7.65209604855975, -4.10990241837356, -4.3379100438872245, -3.144791234860585, -4.257400003418622, -6.678114421469913, -6.866487433947222, -7.697728752690109, -5.8045538215356425, -6.443871311173693, -8.079096309219214, -8.153204281372934, -8.17767713789166, -8.353264662504271, -8.864090286270262, -8.820319623214933, -8.693771874337664, -8.686253041923635, -8.948055666387596, -9.842684901480572, -8.909864867655205, -8.569719225667685, -8.285794202093616, -9.08611589448313, -9.094531091408415, -7.571541334321345, -10.622843459030147, -8.818180584966184, -7.823031278179709, -8.265917771336019, -8.826764328657575, -9.481851947393176, -9.0917181500318, -8.558120089824332, -9.233307603986084, -10.138335143581529], [-6.352576341887475, -4.375482811098582, -5.097247890129581, -5.253867383491415, -5.300921501088017, -5.719261065235271, -5.695077377144448, -4.664604980788103, -6.0509261540753565, -6.1211148128546835, -2.6955840583648807, -4.0885439717448415, -4.601612684872268, -4.3272866151564005, -0.8335121453767717, -4.368698782518719, -4.841722524158204, -3.6648788033138118, -2.5172646408008483, -5.8838060715240985, -5.03345350892465, -4.275021730152804, -4.6405776530763765, -5.25643229403688, -3.133213396702524, -3.881602832644344, -7.333792367288475, -3.6375800962202542, -3.280343679384232, -4.168603006615407, -3.0158762344692915, -4.957870530657444, -4.673357005846336, -7.071279326505898, -4.732292586433726, -6.102987428262127, -6.308187629826426, -6.692855118141101, -6.912844895031934, -6.811275201992611, -7.387021770702892, -6.866694309094154, -6.971677700053924, -7.120054614261141, -7.672123196561181, -8.47105299909571, -7.06564216575394, -7.303313817631059, -6.877175818825198, -7.588372692026248, -7.710844848156796, -7.012073355807918, -9.487490679574803, -7.203069557208429, -6.086293297912648, -6.450936411500557, -8.098699438256325, -7.677018501414594, -7.313496820953744, -9.611543328244782, -8.931691690166081, -8.112931574673333], [-6.278777370845214, -4.701340011999448, -5.078865738235898, -5.8830885176004895, -6.165050673343658, -6.274964098411205, -6.212416394644692, -6.400778791567437, -6.741692863598321, -6.816576094912807, -3.3928904712107735, -3.566247086795751, -3.646180662937257, -3.523693462975528, -4.243767227003855, -4.080529251966207, -3.795135190780714, -3.596123418433819, -2.7581828008765474, -6.336206897605354, -4.412794119233608, -2.7790238541384125, -3.7937492847610685, -1.8714655708089012, -4.039189587014222, -3.917817104421266, -7.57147815542886, -1.7211826197805813, -2.454732555101985, -3.258178620019967, -3.831607640543702, -4.745787338078659, -3.8173867609921763, -3.6132042458306612, -5.788388393117924, -5.078865738235898, -6.509442541403006, -6.358094961721853, -6.982655840165303, -5.955633264954192, -7.203135532877834, -6.519814112808369, -6.508748303549079, -6.5805671825510945, -7.521390752829464, -8.307422019239775, -6.987204748691672, -6.90870144071007, -6.578332663638785, -7.263487211998134, -7.281252691733005, -6.635300404511988, -9.747550691415777, -6.939413018359865, -5.8716986926591, -6.717215468733563, -8.061948046761787, -7.4055504084105825, -7.052394044113555, -9.471951436870118, -9.440772718621666, -8.392393495069179], [-6.019277337678054, -4.809610930457335, -5.353723551448762, -5.676382755872694, -5.70371603264992, -5.986064759126338, -5.8139317591377235, -5.978144349530662, -6.042881895447475, -6.198360420377549, -2.2921047448703056, -4.34095023979577, -4.345648109454055, -4.367598309292761, -2.0155741702111323, -3.0132780221373907, -4.923111941566577, -4.563679111444075, -2.223531060167143, -6.691102686207805, -5.506010794027101, -3.0594865563826104, -4.849221407410736, -4.504665385487454, -1.9294270482862954, -4.87704758753489, -7.8125932274537675, -2.476262613634102, -3.9821539473427885, -2.8193399945915623, -2.274864811699418, -5.85738068573614, -5.424700477670406, -7.189965927648669, -6.389275800912233, -5.638934468614081, -7.03745392086323, -7.341255375194895, -6.260863632640313, -7.279063354938447, -7.679061834829245, -7.811259004440631, -7.236144890792278, -7.5275849533863886, -7.494018129747305, -9.1181262319292, -7.907769904821474, -7.576977708715964, -7.061731490444577, -8.083943001314976, -8.105182737815888, -7.709906510180343, -8.65367988774833, -7.560240916360441, -6.676421938949557, -6.335428678009885, -9.006382193489584, -8.143473650809202, -7.5376758563683515, -10.154666091954931, -9.349927845986523, -8.656780665426579], [-6.45948206141902, -4.541389017548509, -5.0469147172405115, -5.714893702071352, -5.93765911006474, -6.159588168280474, -6.484829040506417, -6.54643138634695, -6.777423978144398, -6.472536180151478, -2.0992585177991696, -4.351360197286945, -4.810779680772453, -4.478347221038041, -1.057979923268165, -4.503803429976553, -4.369600328148692, -3.1761274441594316, -2.5925706700973334, -6.132316774171815, -5.0527495258256305, -3.548319221739584, -4.665298532220545, -4.239092161682817, -2.8823289365699227, -3.9799558061803935, -7.613527691788449, -2.9149052754992715, -3.0856952412535055, -3.988705873974733, -3.519423658731669, -5.380012776918168, -5.236242140888087, -7.294418265937878, -5.378236234528395, -5.653900468202214, -6.843339306558724, -6.1506932535462235, -7.23734725322546, -6.366553451469326, -7.364460455464448, -6.868708500684183, -6.652923233094938, -6.285920796794185, -7.434940854111734, -8.332992180665768, -7.045904469629325, -7.3935637933916185, -6.8666551118723405, -7.771896007559911, -7.6868589648739984, -6.982133113853507, -9.79784886428876, -7.098382406877942, -6.05045863483477, -6.643853499354418, -7.8988076492249855, -7.62952803313489, -7.2191872449632415, -9.030593711575094, -9.78510983851133, -8.512650620040239], [-7.312010848930209, -5.9704021836204655, -6.3878250976208735, -6.964626238636961, -6.946983784483292, -7.4124339237241115, -7.3181741966378775, -7.680384414313297, -7.712878247789743, -7.554150900935073, -2.446039713635401, -5.2426041270797, -5.947299500896444, -4.663409565765878, -1.9484553212571183, -5.736061416021277, -5.712018325729332, -4.9933347157551395, -3.124667578606162, -6.881965531986077, -5.837272445819192, -3.4071965596735665, -4.466780007835062, -3.7439634287862087, -2.4245394005655605, -2.7481043832730636, -7.96925132102691, -3.001717011027925, -4.3613498567188325, -1.0557054893757776, -3.70597885137566, -6.306264213716016, -4.571775536818785, -8.174476930790583, -5.896948417633875, -6.225037976827956, -7.408704473732679, -7.482057817244962, -7.278478028822127, -7.6772808871048746, -8.27951403118039, -7.874318428531343, -7.723948471543991, -7.700568997316141, -8.91379761462879, -8.048158534515101, -7.944081716081054, -7.980577820654186, -7.290733450482923, -8.159308709317411, -8.547715834521, -7.821812931153412, -10.179602578494553, -7.600205085285631, -6.935111692528849, -7.541317747952087, -8.80008790436004, -8.364689804291782, -8.08874001915325, -10.681459532277783, -10.11675467483489, -9.167510387022356], [-7.888212115175222, -6.644858293408979, -6.899926832152373, -7.644887076927795, -7.6845832447435445, -7.532598742461448, -8.35541557609491, -8.047237610771814, -8.039272966898704, -8.151069111852731, -3.668932909455333, -4.782855975285661, -2.6659061839419613, -2.8967154778709228, -2.0223904901109755, -4.446771913881909, -3.000219123980515, -4.8870757508472185, -5.930891109758962, -6.823383785692898, -3.2455278628878323, -2.6715253337166462, -3.4540042231373063, -1.613293110190054, -3.32378892967727, -4.249589417392827, -7.22956773062339, -3.479217249500404, -2.6394006861426296, -2.505331420610824, -5.608431497999796, -3.9128781321838506, -6.817651617834854, -5.899129031817539, -6.838124773860243, -4.798082958969038, -6.707198867504101, -6.862144714309829, -6.946702102337892, -6.119321735858142, -7.3106242225785305, -7.047703884629593, -7.1735103563413185, -7.1142937780684905, -7.8727953781736435, -7.684274173500164, -7.105593531784784, -7.043298950514618, -6.705339482889372, -7.153500510194026, -7.545964351744817, -6.429130146514012, -9.208169648866278, -7.146978747947639, -6.381569697965211, -6.863775814304458, -8.274721978073412, -7.87653370028425, -7.678419897035876, -9.611509357662127, -9.195484489338961, -8.64028017837942], [-5.479179331884548, -5.058052917950845, -5.462604366790335, -6.042749073104725, -5.955329948912708, -5.958323963125313, -6.182592584875092, -6.134964535885837, -5.987223312688662, -6.092926822690444, -1.4103668710151773, -5.3350267357418115, -4.970584606980225, -5.289457186272422, -1.770010358993832, -4.901554513033236, -5.37688758694302, -5.089624713826659, -3.3520314691389803, -5.392942008652894, -4.734460344193294, -5.4364649785873, -5.373540303482963, -5.098468738623112, -1.787587681664556, -3.2734283276295595, -6.018573842572738, -5.287155689284143, -2.4508369623232906, -5.268931950327691, -2.1042922431924556, -5.627005270227098, -5.60465202444635, -5.974953220096848, -5.917202163476144, -5.89303609762896, -7.4048682036926285, -7.92851451588768, -7.027392005486696, -7.609083745121319, -7.38594019380711, -7.373517673808553, -7.797674914928869, -7.69894007424318, -6.9721762828842495, -7.92851451588768, -7.9072371174403955, -7.498269378781029, -7.24617825143195, -7.836141195756666, -7.972477639308797, -8.030297210197622, -7.424161406627308, -8.130380668754604, -5.985681292336846, -7.548459123304884, -7.876146530370365, -7.886403030537553, -8.383576565135217, -8.509870290459508, -8.104405182351345, -7.3612475812167375], [-4.615704248273826, -5.059874046144977, -5.605352624766998, -6.125932548297868, -6.262849675716521, -6.79239877505316, -6.990929985784472, -6.841047905444769, -7.1584533214794765, -6.9886390340379165, -2.1061188583709898, -4.694960766195293, -5.294767434713961, -4.877455498156362, -1.7372915500183557, -4.54913156517679, -5.204152324138941, -4.129384627657894, -2.037945671503165, -6.073264053143508, -4.589028985169175, -2.990753289694604, -4.778969963958759, -4.689721987339014, -2.425686266522293, -4.746481928875957, -7.836127606736178, -3.351158263461062, -3.2418172438356554, -2.1018059989954563, -2.7036646563447677, -4.616611132654885, -4.89196974360223, -7.672674534246606, -4.594182160042578, -5.702759391924034, -6.928687676907251, -6.888038111045066, -6.879281938695605, -7.0247524516915965, -7.464610089759517, -7.314413547152238, -7.29014621148154, -7.293244587314209, -7.623992355607218, -8.372647679876952, -7.108856380340105, -7.38925322665568, -6.931385277679899, -7.915280634635728, -8.074744053353632, -6.836124212582984, -10.164407149105006, -7.372318020898097, -6.283267109907934, -7.0071153102054895, -7.702596214111655, -7.848216404055182, -7.332806708624255, -9.549591811814969, -9.295811291038868, -8.128717117098297], [-6.86596902684244, -5.052082122851932, -5.150933488599681, -5.292208274141252, -6.214682697483703, -6.745331904339627, -6.68552490120206, -6.760045968476806, -7.227305715576431, -6.950863382929711, -2.1215717821722215, -3.9228106947364623, -4.840908762978201, -3.0664686008437925, -1.6983460154315366, -4.061987675375524, -4.329017422576037, -4.468146834761754, -2.0559567932823986, -6.284656471404593, -4.495653917304999, -2.239702601986903, -4.059167724800704, -4.675038779649125, -2.554078005507218, -4.208324039380938, -7.846798279854166, -4.952711862629422, -3.1510988852700317, -2.942373372199169, -3.3895078376634635, -4.604176621456193, -5.1138376420493366, -7.9144140567925305, -4.640693910453494, -5.102920313867999, -6.555881088992486, -6.826369644359554, -6.799686147757116, -6.863359893400418, -7.558312671802701, -7.067443904093005, -7.193279183729173, -7.1899393531824245, -7.705592764618944, -8.457679157012771, -7.256276285899237, -7.186610639983997, -7.0503828029559665, -7.591798854832744, -8.06246715675167, -7.178062400973902, -9.768878866076495, -6.917516203514602, -5.665418470446385, -7.260143611855673, -6.937390775844807, -7.687607497719584, -7.3485921408206325, -9.2384661090352, -9.464847655623185, -7.897922355187247], [-6.774920173861653, -5.204290861824316, -5.43566671911157, -6.296930641076284, -6.565626027570306, -6.7373590105822805, -6.991252066666452, -7.063233360250443, -7.2676834795551315, -7.1966044322333405, -2.083909994196951, -4.067585265323549, -5.182321460673428, -4.835732283228186, -1.980079058156674, -5.0283663440075825, -5.072641431371989, -4.6392056435133515, -2.7086094993416667, -6.770304135451181, -5.008181972697398, -1.1019554288598505, -3.2594369265598493, -5.6765883311043766, -2.736784393019736, -3.364824539297133, -7.918041582638827, -5.789527217909832, -3.4282422713486094, -4.498435097687583, -3.6322465613937016, -5.355734234640129, -5.0804532195051415, -7.17597860187687, -4.9759313787901425, -6.718299350460867, -6.452973839496266, -6.266241964550636, -6.756169624516277, -6.43275589071587, -7.149632742116933, -6.722481373881418, -6.632646871978412, -6.5830057991087605, -7.441299494170494, -7.5325638154552745, -7.295254585596671, -7.196283868037615, -6.683920964952739, -7.4835386813502325, -7.587428922371031, -6.992558402580507, -9.54477866233574, -7.0691430439756715, -6.0008086042221205, -6.0079129112663265, -8.273962947039259, -7.457398913104675, -7.183861348039058, -9.235518989239411, -9.247910721534573, -8.527701618931673], [-6.725963719695806, -4.710012269685353, -4.981543035572536, -5.7693914443296395, -5.741238652017211, -6.34104225410854, -6.461701921452123, -6.616580803658851, -7.050213688876461, -6.932988782385651, -2.7442254377569215, -3.902092293780571, -4.081087684032349, -1.8459531867558991, -2.260196124051401, -3.8113055093318327, -2.1904142607255017, -3.5077174805325915, -3.070625904070985, -5.688404814159279, -3.676818033328958, -3.705969932950152, -4.175966474127823, -3.555777846153588, -3.6599169276373167, -4.233380646142029, -7.565625193952616, -4.758247211588438, -2.668505149650637, -2.593317887541856, -3.982238613532973, -4.922572014698711, -4.457568202222798, -6.922768742308341, -4.874097132395777, -3.7366670217502587, -6.099051002533345, -5.950541812233132, -6.754321355464054, -6.000835251677606, -6.818602573551164, -6.538250446625281, -6.5136752899401404, -5.7389354920585625, -6.837502673710269, -7.922940958163197, -6.4879709082904045, -6.559235554477551, -6.415308634074, -6.963830229106114, -7.464928001069955, -6.466545899251641, -9.489635645899149, -6.3635319575989815, -5.527598474760337, -6.407003965795449, -7.463914829757348, -7.418626873140783, -6.4607730713017455, -8.850715648219035, -9.544876513609825, -7.984492491801492], [-7.251231594323855, -5.21374696908488, -5.698223064924641, -6.1375976604763816, -6.322511629217009, -6.768362468268195, -6.494106816635844, -6.457670861321346, -6.860741199932812, -6.755693520691695, -4.0580333694338515, -3.9411669781297105, -3.7467312772977017, -3.0227837077197126, -3.61529958808048, -3.574469853485688, -3.2007714947116837, -4.080705573827712, -4.670471138732819, -5.7966968651863855, -4.155602316322404, -2.9527040483599185, -2.862751521261682, -2.037725742388079, -3.7818488691194445, -3.113228390392548, -8.164264994327704, -1.8458475827404217, -2.5467378386179824, -2.7020967605017185, -3.188124830525815, -3.6799312814828857, -3.4909044267832696, -5.197365837878935, -5.202188998354364, -5.870321468395694, -7.4721345906728445, -7.508740887837122, -7.308294052687071, -7.524695844262695, -8.528908107915612, -7.92499206108361, -7.684820971302881, -8.246116020758024, -8.52404191826444, -9.16793672064164, -7.835273241415741, -7.178620119286367, -7.559608480967977, -8.499112535221915, -8.875184344633446, -7.775193695995215, -10.198665810583236, -8.01769231637253, -6.730988787795164, -7.673144441641045, -9.323628945401888, -8.330457169191774, -8.013022138590646, -9.77099654516406, -9.908396414566504, -9.416350685452109], [-6.383306695601549, -4.903894061372386, -5.294037540593016, -5.450500954330478, -5.333223053427145, -6.308171725769449, -6.710407117707228, -6.887542628987831, -6.847572675445971, -7.035457528925254, -2.2105034037003235, -5.0202198363584465, -4.228806005807516, -4.2610399361561155, -2.630714925229401, -3.899363165669206, -4.911168632565807, -1.9598915311908123, -2.357698156549484, -6.874278408705108, -5.4856719078268386, -2.9453898821076563, -4.76865138002776, -5.849682782165729, -2.1582691520417923, -3.13711767038794, -7.932068702852963, -2.1341546758789423, -3.3759594869296796, -3.525124340216131, -3.4812218611704324, -5.083766155674364, -5.7936328268294535, -4.353576622920111, -5.907706290260866, -6.135534326871947, -6.846532994187875, -4.609063889576182, -7.503287789431182, -7.483445868632177, -7.2287173521187755, -7.853210127940644, -7.155334806184752, -7.407188463381393, -8.090673733029602, -9.081164896953064, -7.857960730699241, -7.896800564015505, -7.262268886290009, -8.128645226782451, -8.177248142566354, -7.485412437604219, -9.720755973044048, -7.9739596445621235, -6.796539470357211, -6.907958941482774, -8.646989682962468, -8.283628546124925, -7.797896489956591, -6.6686665609139695, -9.771081056931564, -8.750720965163202], [-4.685181991517679, -3.1394221383869034, -4.505733999919456, -4.7128201143378075, -4.721709061755053, -4.800636320344724, -4.867216645426291, -4.85864761270119, -4.820149134568306, -4.881081685563463, -3.6610088191091923, -4.523883318425133, -4.405422178721609, -4.511747247142908, -4.4973757423052545, -4.410859930021738, -4.692381446660533, -3.970040309575452, -4.061847858828575, -4.724689689893191, -4.678033998252392, -4.350556098635668, -4.167208374486401, -4.450909776342023, -4.644417387453407, -3.724277324458299, -4.630737283549326, -4.509337607422754, -3.787718752781497, -4.309323510802315, -0.6823718671879156, -4.449775346271411, -4.306372205167735, -4.410859930021738, -4.920232593125288, -4.3914208729895154, -6.478377211171838, -6.487035273914953, -6.801150603434889, -6.531487036485786, -6.875258575588611, -6.039010551387992, -6.9553012832621475, -5.362670489144429, -7.042312660251777, -7.027274782887236, -6.983472160228843, -6.58757650313683, -6.732157731947938, -5.820321350423163, -6.504579583565862, -6.710178825229162, -6.395685495326725, -6.710178825229162, -6.559138567816296, -5.802621773323762, -6.789316145787886, -6.8131267944816045, -6.29749346914751, -7.1889161344436525, -7.012459697102096, -6.888161980424519], [-6.973841065843466, -5.090695336189662, -5.341561221116127, -6.0115634498929555, -6.202945132409144, -6.628849553394376, -6.911669409716161, -7.0009992844957765, -6.886081347969349, -6.711294966663227, -2.260331825727248, -3.839334077735597, -3.9667628422059718, -3.420241689905642, -1.8735787833427253, -4.188499103605226, -3.691171678101274, -3.8593630820739957, -2.3524981891706576, -6.231341006564324, -3.7662373591100775, -3.6465568924129386, -3.4963200738799767, -3.4321214183238204, -2.593050419274572, -4.346868167488547, -7.8364379197408365, -3.990422275679548, -2.904818352752614, -2.3714612152432646, -3.1378637759089556, -4.357037863124861, -4.651314292504279, -6.760867254793699, -4.107313187039242, -4.3976920060255535, -6.440759053046417, -6.465477098310146, -6.582182413654962, -6.610599862587676, -6.978263755093254, -6.719449335464573, -6.730879435939417, -6.514035039942995, -7.843934206718764, -7.66305587798569, -6.780807453646412, -6.810345466083276, -6.347157031161333, -7.321885120582923, -7.779113558339937, -6.602957069625781, -9.658525717989464, -6.893034874465941, -5.640541915214837, -6.61808064554899, -7.825297811959789, -7.623352248005446, -6.729273803531229, -8.625119353004427, -9.394209194493895, -8.033505455437194], [-5.604855175474509, -4.462285490693087, -4.571350651692403, -5.056493889833258, -5.072440143885144, -5.681064638387167, -6.280536665394942, -5.74786868753166, -6.300402104581103, -6.443541857970619, -3.08781816164275, -3.6971443171127936, -2.396573090959646, -4.216785762291264, -2.055069601332964, -4.2400357108180975, -4.510301966657801, -2.8725223515174108, -2.985980079879496, -6.151031964062443, -4.17853136290412, -3.739735408911567, -4.177743865892257, -4.758224336321508, -3.406169247786832, -2.7407038589797272, -6.9336687884910315, -4.884332887441078, -2.4403221799860293, -1.8263508274951405, -3.4054696763623764, -4.562592776233452, -4.420217843796468, -6.782722037918705, -4.9372619598181355, -5.457485531187882, -6.256287610141627, -6.611108985431037, -6.565907361549403, -6.49534726380481, -7.061567128112425, -6.599672447239019, -6.7013251912547664, -6.80746927442344, -7.516987128232732, -8.06324629910308, -6.809992050813993, -6.860502751231873, -6.646463246903641, -7.480934599971211, -7.54322475844959, -6.575185095427639, -8.985698479921282, -6.994899040702809, -5.767168821265772, -6.530694737719788, -7.839611467995152, -7.435063610819411, -7.091053743823944, -8.707954377459634, -9.10781723001913, -8.370947238445352], [-6.890344120648169, -5.02940421086846, -5.408867487897182, -5.967857793669728, -6.047653312106489, -6.46237897058548, -6.749954722267399, -6.888388607925276, -6.98955227182629, -7.21719797180092, -2.4868466816371, -4.771074471877903, -3.6481121610473157, -4.664656454469179, -1.6817234068168874, -4.586866350603287, -4.845552155508254, -3.154508356873059, -2.57905575846229, -6.254276196451527, -4.921506816489567, -4.4705470709445265, -1.6402347840968394, -5.1280286653720175, -2.882844123251368, -4.595878517780293, -8.231899529453244, -3.131328577959143, -2.841674626039629, -3.3059284308174086, -3.1776616460738145, -4.478455963747706, -4.881954395151054, -7.651052083171771, -4.569886902888001, -4.27044926534142, -6.859624364774909, -6.620591583748609, -7.035724951332805, -7.0226528697654516, -7.663432064808081, -6.880725735179707, -7.096836038369936, -7.001238775261635, -7.94609099384926, -8.742881147291776, -7.2506609077408095, -7.3153280695559095, -7.053568548019118, -7.728589206441718, -8.033869830584925, -6.95515998225557, -10.250083546611185, -7.317576103473792, -6.34314386192865, -6.83930777142871, -8.443181097280567, -7.8493114938202, -7.476614929901965, -9.60010853791645, -8.54213954241637, -8.677952686589895], [-7.97530600380511, -6.415691397743754, -6.861539964259856, -7.354178676035024, -7.5544322703441, -7.956843940965374, -8.00285595557334, -7.941715059369074, -7.997284910523884, -7.042078762400056, -4.593656623473984, -3.4358427231129607, -2.813804032113097, -3.8443096208464707, -2.253236786554918, -3.523372060796657, -3.771274119941123, -4.213005093082092, -4.26676474950711, -6.604661414352346, -4.071781743539046, -3.0636392294004997, -2.748766550188664, -1.5979085155684218, -6.096125476162904, -3.8319741165296572, -8.437883856353995, -2.362166563210972, -2.079080404566457, -2.423752746988421, -6.1373971409737065, -6.112261605870961, -6.033619491127531, -6.187046440930159, -5.880551407088361, -5.508879098944167, -8.771401153617497, -8.582218506443597, -8.081702252336468, -7.820268405959624, -9.01805471242806, -8.991966276343762, -8.037776271878622, -8.339328192194554, -9.20125771484982, -9.932145223392613, -8.711282886564021, -8.633361864553784, -8.75433589306331, -8.724333642759511, -9.49070501282617, -8.37573721938598, -10.194509487860104, -8.552742688310644, -7.968342726152964, -7.956843940965374, -9.863152351905661, -9.598817643450872, -9.08877973142313, -9.278218755985948, -10.729432663205156, -9.20125771484982], [-5.1527222081970026, -5.012025127950751, -5.39388426501389, -6.217687012129963, -6.355002880531065, -5.6351154829069525, -6.363993950090922, -6.903492332891861, -6.853964796193012, -7.129436704737862, -2.71361594954537, -4.917813116205636, -4.909342477322127, -4.834757647401019, -1.3606202815267094, -5.049186428683174, -4.831934627067998, -4.02878090419557, -0.9857025802248848, -6.9583248115172545, -5.928056254423655, -5.1548164498001166, -5.532591229650075, -5.575217341325883, -2.1159432194090155, -4.2897086796604516, -7.362881241689201, -5.200780422733714, -4.007732553855012, -4.899317914996637, -5.732412626945404, -5.409409253401815, -4.570479874572386, -6.004563148213606, -4.806034206660294, -6.602559371961363, -8.008146936015933, -8.264296783798311, -8.114951495245993, -7.067733915701082, -8.653122572902513, -8.40456220860418, -8.35558820400772, -8.362968311305343, -9.101577865942044, -9.242891399008622, -8.693532111240389, -8.468328826963168, -8.348262163915647, -8.71436619814323, -8.123659823137777, -7.872345394856871, -9.1093600063841, -8.693532111240389, -7.279281672853908, -7.536521246398329, -8.924437667890087, -8.99854564004381, -8.061492916721225, -8.91796515338447, -7.579574252897618, -8.90514446495541], [-6.625065979493934, -4.788580474517243, -5.256407824340383, -5.37832984858721, -5.2038978790684185, -6.079823689638168, -6.317086566978479, -6.646320764762257, -6.72855886299923, -6.886116701148103, -1.505097000684665, -4.82588581184553, -5.253602398681012, -4.989577089424305, -1.5758741362397704, -5.185784176269271, -4.916501452893276, -3.8159863951228723, -1.6041923545980379, -5.633841418592661, -5.7199839934992545, -5.2161585690547705, -5.369003667873634, -3.3721189713516244, -2.661522576130661, -4.670108701549678, -7.227790153984301, -5.183916064386551, -2.494756625753895, -3.869077211017653, -4.27104038948766, -6.0954898063825675, -4.71167529774246, -7.136943681210728, -5.987593808088792, -6.590446143960144, -7.403791666759668, -6.814987319540992, -7.030531087602099, -7.451894781778061, -8.262224216961, -7.8389014914528286, -7.199379152151521, -7.871278420177289, -7.002535480112671, -8.630549778119708, -8.06204504276704, -7.169938203345767, -7.199379152151521, -8.169374062997537, -8.264930578558742, -7.090662928646722, -9.550754409315, -8.033689817011913, -7.078178371984477, -7.674437552058881, -7.921898411023797, -8.116604027017473, -8.273093889197904, -9.39660372948774, -7.831846788472939, -9.063739434242727], [-5.33959692367457, -3.901875433911095, -3.3263523840890916, -4.682859878096868, -5.0569946532038434, -4.833678156338227, -5.439237690659609, -5.4278954140556746, -5.662170026427569, -5.462315637942154, -3.751806551579704, -4.660118258928943, -3.8333548860836233, -3.8811153084238335, -3.4958051088141735, -4.553750303632283, -4.534932311997922, -1.8458046740483725, -3.0281790617647277, -4.436160927561686, -4.943995470762378, -4.549402848736927, -3.9218705969950047, -5.328568099202016, -4.452091297246621, -1.4302390343932077, -6.080922600826194, -5.1148485422425125, -3.3219189169449392, -2.621905648275213, -4.3789508674983555, -4.262632175345694, -5.088920047216344, -3.3849839728869298, -3.307281062205608, -5.974439120423745, -4.379829216259262, -6.791884017661267, -6.8166991867809905, -6.068955964208674, -7.1565271312491765, -6.683586642330131, -6.801736314104279, -6.532793078538745, -7.518256845849533, -8.152370494825284, -7.391505140210389, -6.7194113688482755, -6.416271872702784, -7.279027156783699, -6.4871905993829495, -6.66039719741122, -8.204669994228134, -6.878895394650897, -6.094660190607558, -6.427558674237414, -8.084652320770335, -7.339319387039819, -7.347830076707727, -6.950948712290955, -8.303341521735163, -7.468823387994359], [-5.431835629645527, -3.900968951139421, -4.151004765492447, -4.646528675532386, -4.521804796114552, -5.283128043243618, -5.29007437093391, -5.66335521876192, -5.740643416211578, -5.366660147370001, -2.670098748915231, -3.286242237340954, -3.3585141055181613, -3.755602707715123, -2.5586532961090027, -3.6926656242369105, -3.8330192649391925, -3.254475467118251, -3.8290307266838655, -5.019290301102854, -4.363753678748955, -2.9684754808494547, -3.151618081678543, -3.50657051212598, -2.895083629045828, -2.91665360144011, -6.285849431998168, -3.6228229122699127, -2.152751777383897, -3.3162167836347605, -4.2488413058475105, -4.652746019284857, -3.9288228848114928, -5.349158624766704, -5.572231075081914, -5.261791793324819, -5.624778734167247, -5.278347157643276, -5.3043854022704995, -5.210237912616575, -6.346428699255328, -5.93048158286466, -5.615084400228674, -5.89255288484716, -5.916841563358977, -6.777270925569916, -6.237681480853062, -5.483828321441287, -5.599254877723378, -5.735607736245962, -6.332645938959907, -5.443956990177872, -7.787390135984574, -5.974679054270741, -4.487700164671342, -5.484479363130949, -7.787390135984574, -6.624646575217306, -5.915838554247808, -7.256570295689133, -7.872330812633658, -7.3300889926342885], [-5.8962959205599565, -5.022482812723318, -5.262967235210922, -5.638331471894197, -5.778457744433211, -6.3109845765456285, -6.29163873510587, -6.56832765910669, -6.49255510063436, -6.3650100589758205, -2.863492599093995, -4.089024629780227, -5.154949557468251, -4.484159695248834, -0.9343644420039627, -4.498161263246394, -4.6279862283262885, -3.982370058053253, -2.2035282537140866, -6.199482179547703, -4.571665803362194, -4.084960905072187, -4.755413554835734, -4.800997636581884, -3.6971982708382995, -4.566795129611478, -6.871351961660363, -5.099526707333125, -4.135788994836005, -3.1706024468180507, -2.212641008744242, -5.078715683958774, -3.5811841558979225, -6.741940888008861, -4.856098125468122, -3.980907107020626, -6.712325967469931, -6.644006723492454, -7.153784678226001, -6.945055157447498, -6.898020208742524, -7.021563043483541, -7.173168545047049, -6.854391346353318, -7.669859657878135, -8.284314569862415, -6.895321151773359, -7.513205847832758, -6.795366054682441, -7.556635405760094, -7.827916592672458, -7.241272132349116, -8.35137880044296, -7.187505708193456, -6.1188193168155305, -7.139919638088829, -8.075324765986299, -7.379987004046535, -7.364785842714484, -8.480789874094464, -8.527936652520165, -7.717773013574032], [-5.1596391409467905, -4.446625879300866, -4.792661465824471, -5.092676376197713, -3.486026851337387, -5.293817992318437, -5.4207097441730605, -5.031624556083574, -5.433394903700377, -5.123410202650332, -4.715036303793935, -3.7021536789683007, -3.7870302329121537, -4.088809455118969, -4.812132309601868, -4.898021981482298, -4.704305701448848, -5.045386241156256, -4.694501701352228, -7.439823297251669, -3.3979941838940024, -2.7387677229303073, -3.4751598040428164, -2.0375451283648167, -7.179408351140228, -3.811103778633784, -5.971540702552751, -2.9729698958508197, -3.825149191015764, -4.478865937874288, -2.6641628726801234, -5.3193066285353545, -5.996634270150407, -6.375549146164451, -6.785390628872414, -6.572829186888741, -4.124487206195906, -4.535138151336436, -3.7135200209337587, -3.96463850525401, -4.554386339641352, -4.765674647825141, -4.429638997109137, -5.277664924510665, -5.02086262836912, -5.937153360243275, -4.7565501004863675, -3.693750268171921, -3.749678820831146, -3.176252392796863, -5.722496422806384, -4.546642548744449, -5.417354036326088, -3.1803577086920454, -3.749362815140414, -4.066652137558128, -4.660532401294013, -5.1570534898617675, -5.430844966067104, -5.284241491875228, -5.5748604240003194, -5.63236491848051], [-2.1543734650260515, -4.349659150265929, -4.518901599180162, -4.334150339570116, -5.146931928181808, -4.726677895862385, -3.8738862022041083, -5.547738126032732, -4.965611539500847, -5.329386916158828, -2.0510193107742145, -7.017067715762613, -6.5514024881154755, -6.782731624016124, -2.1387030138865364, -6.887331394106641, -7.26098839501858, -5.915573580719614, -2.792353319034718, -6.475549145597215, -8.196805549243091, -2.7290425937620233, -6.57338139483425, -7.482751683863832, -2.7069370493135643, -6.6217549791184664, -7.959675755954141, -2.6929063910157374, -6.402701413055781, -7.378081061299943, -3.038722067403394, -7.146279447242619, -7.442205589469482, -7.777354199160186, -5.436617340201765, -7.9268859331311505, -4.21410095816366, -3.7159537926574395, -3.274052690014408, -3.8709022387814955, -3.903841048493916, -4.907248138723605, -5.564815386850924, -5.634025390029099, -4.794669640998671, -6.37044055083756, -5.773624479216047, -4.984713243886536, -4.705003878814023, -5.884409585684786, -4.869643914101749, -5.8105008762596295, -6.426399204485604, -4.552569937605819, -4.688840192155229, -5.1496039404596194, -5.387063525747035, -5.940338138344011, -5.40158748887255, -6.8282736444630405, -6.302506440698851, -6.49077786629904], [-5.042027972011844, -4.299718672656642, -4.243324982365076, -1.9825142998026875, -4.725280240738409, -4.751810055033314, -5.528415080508104, -5.372269658882421, -5.58057814379269, -5.634199235231788, -2.092585085927561, -5.359310514239916, -4.924244936355461, -6.182928172882705, -4.358300013368391, -5.644135441890917, -5.3193472574280785, -2.4216711986248063, -3.5454438576615357, -6.6553797837670805, -6.406683158028254, -3.42063060974259, -6.230496589801815, -5.313180742512414, -2.1141313157147192, -6.38251279710044, -7.199431055161392, -3.210947634389801, -4.823896962521179, -5.599497627038855, -3.7500872171087103, -6.903338504201014, -6.960201366095558, -7.542682978767983, -5.613234106766742, -6.36476085164199, -3.6714819678600925, -4.875454772186028, -4.20471280907303, -3.884017303250544, -3.4828969984685907, -4.20471280907303, -5.449056802096191, -3.411103801364231, -4.476181634325785, -6.299489461288767, -4.386290645201218, -5.042027972011844, -4.789932598747867, -5.333885415874105, -4.1803254304220925, -4.950988261330972, -6.251222720318932, -4.73156059252596, -4.355150404465494, -4.750641830601388, -5.174666905853347, -5.78646844695934, -5.852762894799176, -6.180486169227153, -6.353098911894147, -6.096324376905443], [-3.621153540136533, -3.927545190670303, -4.761298021301781, -5.2327070159671, -5.390836480884121, -5.489330016767771, -5.419938023343771, -5.638503085521277, -4.333907884333748, -4.5925613698517225, -2.3774699118392917, -7.240027999198052, -5.856199730573355, -5.309006462636489, -2.1188535037876086, -6.988713570917146, -7.259929153515347, -5.733217613565709, -2.1988937611323722, -6.914879308906094, -7.285375819176511, -7.471829613255376, -6.297483936486472, -6.369085087302357, -3.0144535413397078, -6.131611710433321, -8.510182710405289, -3.290870578196505, -5.79050447130796, -7.191941812530414, -3.661034886200146, -5.933160771709483, -6.992552347224311, -6.430741168725453, -6.006448567479316, -7.6405790925037875, -3.820123280601132, -4.60874181430768, -4.552214016916188, -4.152125265844404, -1.8440074384079603, -4.33229280995845, -4.748982594711727, -5.590043992369601, -4.083551769494654, -5.885005729822436, -5.570371226770897, -4.35431953325285, -5.037889407059404, -5.600505333614971, -5.2234842891983275, -5.239347146606377, -6.012203979049936, -3.2506791222758458, -3.493367719618491, -4.386449689183688, -4.5929103490909995, -3.6635121789617022, -5.259536257638402, -6.071656848963408, -5.913358144413303, -6.637031915632404], [-2.697794466436189, -3.604662393389068, -4.011263957500705, -4.728080882146555, -4.781372179196779, -5.33155887684335, -5.439355447299057, -5.586922158039964, -5.370633615250003, -5.513150772897161, -4.398260316869362, -5.11207163673225, -4.190082883743458, -3.4501177344839546, -6.597640067453566, -5.490434815062624, -5.150911470048515, -4.837107457567751, -2.85030853825846, -7.320346050255055, -5.997701716211621, -3.4753179136511876, -4.2866759373606715, -3.397053452568678, -6.703360230660674, -4.113003141200578, -6.444877312901155, -3.074043686998346, -3.452623303723489, -5.193702907090925, -3.9327969395172757, -4.131242383720524, -5.418238523858134, -3.7478131640115406, -6.184652086682414, -7.063123185287982, -3.7394230709797665, -3.4179694218787144, -4.267288710081795, -3.0709940324857143, -4.263518677341253, -4.701005637323521, -4.417094629495229, -4.569236359692398, -4.473887479448818, -5.8432973258667005, -4.728080882146555, -3.754953367214181, -4.1263095154005, -3.415012032977706, -4.81618158386264, -4.691820172692138, -5.666572324000903, -2.795735372125381, -3.2437129699651224, -3.9019638438387987, -3.582993942690341, -5.112804505947904, -4.894364079117085, -4.722110715160052, -6.056219323109372, -5.5022692727096265], [-4.619141531251806, -4.242478414047348, -4.655112011634125, -5.260049492712331, -5.018630839221239, -5.434973330528597, -5.291682300504695, -5.652426966065128, -4.133100006910822, -4.834429582625707, -1.998550183209591, -5.7433143168994505, -6.845817661060527, -6.562531200081784, -2.4549848852751466, -6.626189051853761, -6.290387731057782, -5.567486289416045, -2.141146196531367, -7.242549089199937, -7.104679294976816, -2.8517566389677937, -5.192894636686687, -7.082206439124757, -2.3092018462234853, -6.71448165899944, -8.017189304011065, -2.1705324264442973, -5.663122255181876, -6.392769837961348, -2.584044215625091, -7.28700085177077, -6.816997222525035, -7.454881724409931, -7.498844847831047, -6.3210545294507545, -3.9234738704665255, -5.210974225191004, -4.156261002958894, -4.841496749848799, -4.597299789336757, -4.421170202790724, -4.975365923256807, -5.672123216040852, -4.474317672329112, -6.090327349926446, -5.367408011032831, -4.491383872387231, -5.280618880960439, -5.815780620152271, -4.640826054386648, -5.541761398177608, -6.307308608546119, -4.360911011272527, -3.968855752517438, -5.060904184340462, -5.092811178129705, -5.9935197902566175, -5.119506808592894, -5.086750553518014, -6.052587021943599, -5.38489332537385], [-5.261046281621696, -4.9148297535258045, -5.093521542269181, -5.160962823064713, -5.492794680129424, -5.583178741597693, -5.608843109973599, -5.828958322395465, -5.818217080564052, -5.6351835265880075, -1.9431063436877503, -5.762127613913009, -6.364566555601998, -6.310693565661846, -1.465488331318766, -6.653479397439911, -6.983078754773362, -4.701955198551832, -2.920630198813372, -7.381717897811127, -7.114655112562081, -3.628618850962897, -4.741931970713727, -6.050483313874263, -2.961525855445553, -6.683481647743711, -7.823550650090166, -2.1208799042866553, -6.236585593508124, -6.735576759627112, -3.3232120191551315, -6.891511561247872, -6.08336298799342, -7.9413336857465495, -5.554867108771802, -7.122498290023107, -4.193090909121505, -3.901105949199606, -4.152689701478908, -4.935827900365578, -3.224590180736388, -5.347946392964479, -5.205112607677643, -4.550306897288771, -4.485129674572183, -6.303724896345753, -5.654950567328784, -4.928779413034516, -5.0770940681272325, -5.30867433061095, -4.570963767879653, -4.7737511552658995, -6.146602446129337, -3.6472116840395126, -4.583317229489316, -4.247999881283233, -5.050961927850385, -5.554867108771802, -5.239253651630843, -5.752077278059507, -6.0889495947020595, -6.114482896707225], [-4.8357984388643835, -4.657872493543467, -5.438431611883497, -5.481947249319659, -5.480513557317811, -5.739017089171186, -5.670890393572083, -5.859121927588727, -6.047318448825703, -5.930658377913943, -1.7234534048021994, -6.808376835001858, -7.018097365983928, -6.515303913915201, -2.0800327647225068, -6.985307543160936, -6.434021280478344, -5.690138581877, -3.0546210789668655, -7.433612809945593, -7.609892052283585, -6.846949109788098, -7.145930737493813, -6.771237288052402, -1.3242305561151027, -6.88123818326673, -7.934388097858083, -6.599387031125742, -6.8029859863669815, -6.569147145936024, -2.3927324727103167, -7.200418922777882, -7.33738477785104, -7.62201341281593, -4.893045313052039, -5.366877919534872, -4.1899951004809015, -5.453656819480563, -4.694403309779647, -3.431804500645092, -4.0124147615767685, -5.525942614164559, -5.309719505694924, -5.568828205702649, -4.124028746206436, -6.415604553692113, -5.4275025413513065, -4.984699762805498, -5.237511197353998, -5.157641365469638, -4.101047614464405, -5.085610237260755, -5.550223017871613, -4.869440755282619, -4.489173830779152, -4.234321454353582, -5.041242413079193, -5.780689785571754, -5.231908941805328, -5.964947451392574, -5.752089170738539, -6.139854701747302], [-5.508465007700513, -4.839258034686576, -5.109472414097966, -5.519275923804729, -5.707703038161169, -5.439902336263216, -5.7235763873174585, -5.821556795677663, -5.734300350680434, -5.610247702010455, -5.910709501108635, -5.7235763873174585, -4.435262434557772, -3.8610478472011964, -5.758858201462099, -6.0632537548876195, -6.044700346991872, -4.117878794499082, -6.575328598054043, -7.243783166023616, -6.403478341127383, -5.083829983484629, -3.317712944869911, -1.7163886672099804, -5.622223893057171, -5.543479525545615, -7.373836294271814, -4.235327303674574, -3.4757995308184078, -3.8216678856997803, -6.863010670505823, -5.619817153026606, -5.966922645949188, -7.096625521687328, -7.1845942946332855, -6.5629060780554855, -3.9611313057581787, -4.164449795424729, -3.6745940531223313, -2.809054169189011, -3.303860323066671, -4.356804692686432, -4.075943929085389, -4.993775889109478, -3.5055329961999946, -5.4956443192714515, -3.896020865817486, -3.604273312212492, -3.5622476861654384, -2.789152302491872, -3.9102728885246876, -3.0660244750660355, -5.631909198791635, -4.404134333185408, -2.9033092297609358, -3.155554830818952, -5.387104150696246, -4.341290047595106, -5.258585299895267, -5.109472414097966, -5.821556795677663, -5.14024407276472], [-4.364789554126129, -4.918384854279969, -4.918384854279969, -5.1613310328903585, -4.587530609962979, -4.784853461655446, -5.006160465162354, -5.0403784224726955, -4.994002260682545, -5.402493089707247, -1.4796200682550344, -6.181000119218348, -6.701776073837507, -6.252825853789604, -2.1508860947624866, -6.086590434747274, -5.772240115213331, -5.482914657017821, -3.9652464014134288, -6.426672783592586, -6.401671481387169, -6.285261129542758, -5.890845857621178, -6.3772800282630095, -1.9660163181458141, -6.770768945324459, -7.107241181945672, -5.362398632505369, -5.920923312858457, -6.7530693682250575, -2.0597919147266994, -6.701776073837507, -6.505661194911217, -6.5476253940102485, -6.637237552699936, -6.7530693682250575, -3.9354569646113213, -4.092932522818745, -4.814706424805127, -4.961309898997003, -4.355174095426687, -4.949681861001884, -4.869194610089197, -5.046727650151355, -4.8507000274530325, -5.115948027042541, -4.814706424805127, -4.689097748317353, -4.863875448611597, -5.043547997233975, -3.6732539774605253, -3.9481153614832447, -5.340799520701906, -4.835115296436334, -4.068688911208753, -5.024679512929592, -4.753487802377227, -4.994002260682545, -5.247044551164388, -4.698046353893368, -5.307182909902921, -5.227776132298511], [-5.115394338583977, -4.7561420736309445, -5.007842676853897, -5.66123448019507, -5.033665707964376, -5.645764558422938, -5.535809351386801, -5.907471144471458, -5.828086115216893, -6.105296887801378, -1.6832402388074508, -7.437524027650993, -5.590313253886724, -6.862982589498895, -3.0491034044747836, -5.586172461220692, -6.632458930887062, -5.218606625703514, -2.4710973040085813, -7.398809515470303, -7.268447697607059, -2.7116510959031834, -6.686853002952861, -3.7188303108266303, -1.9217106058640063, -6.098376444956805, -7.548749662761217, -2.711068175455316, -6.088084758920257, -7.1630871819492326, -2.4953178542046213, -6.5371487510827375, -6.531815405107375, -6.592107635363495, -5.218606625703514, -7.464192274733154, -4.025855346121542, -5.098759278079445, -4.812664759541027, -5.369662433402117, -3.845996309584077, -5.031290409935469, -4.627804324836165, -5.549657248245595, -3.554062953084761, -6.151486270270753, -4.8426621708268565, -4.640563668589926, -5.069989313821911, -5.128381534110789, -4.6191257693799175, -5.164313543336852, -5.9721996171719525, -5.146851716173189, -4.482065653234942, -4.222339216728002, -5.190396010216277, -5.978297197040071, -4.681519433426748, -6.057830350562455, -5.619789072019677, -5.749664153441089], [-4.867406343870729, -4.4533740975545415, -4.953434176237852, -5.384635998990067, -5.328952912513263, -5.6593669190947695, -5.597013688377717, -5.757806991908022, -5.710273291442284, -6.117674508577314, -2.0752354231922467, -5.330274789771179, -7.239817294655618, -4.653127950557675, -1.9861891376978962, -6.498730651806554, -6.748830013109291, -5.799220777458779, -1.8880861157511444, -7.060476365999801, -6.592340150928861, -6.615981913985901, -5.98705432616025, -6.055682832549346, -2.2283490142554703, -6.244583360441343, -7.060476365999801, -6.9543698600052535, -6.005072831662928, -6.675112437212724, -3.324941220245065, -6.737960340872387, -6.846328377594169, -7.276184938826493, -5.466076330930241, -7.601607339261121, -3.2284338811241238, -4.7571452826690335, -4.2772171644143535, -3.794375211195705, -2.9321390456478555, -4.655146114713912, -4.556474587206883, -4.658518799192551, -3.9133691163329902, -6.159223511490186, -4.874928318325417, -3.9851607325065794, -4.593769151695071, -4.865742449976033, -4.192747084853396, -4.158562847663465, -5.583291346122616, -4.6687056444995445, -4.403457644910037, -4.135470875670939, -4.338591952168042, -5.484425469598437, -5.10713123845697, -4.666659956776251, -5.507845743806536, -5.56639905255811], [-5.367738120634485, -4.778432275797196, -5.153988612512787, -4.9880684755260525, -5.022992659762126, -5.315533041364984, -5.937669186281741, -6.10637419308453, -5.985149724514735, -6.0666028696576335, -1.4917500304139726, -5.449683054485573, -5.538947192058466, -5.596026010244295, -2.1356770840200325, -7.153812526227932, -7.094389105757131, -6.013018623523528, -2.2775252846627545, -6.010847070010019, -7.246405313055757, -6.755173383190167, -6.697507741340358, -5.530860894627108, -1.9981251565724831, -5.698525293621089, -8.083802102460249, -4.860110306811463, -5.882753428685626, -6.945748081740843, -2.8863534391722627, -7.100819996087421, -6.962461562714583, -7.231590227270615, -4.4224533361891325, -6.0688990819179836, -3.292438554454217, -4.782234561746934, -4.459461169483883, -4.359771877270199, -3.9163352754091854, -5.15214868559078, -5.0863471755948115, -5.622152314836516, -4.041658687615878, -6.321199217523649, -5.22160122153078, -5.082053480720112, -4.742403534262838, -4.896452531238822, -4.577416633798337, -4.325814840936012, -5.22258692885554, -5.570564946046022, -3.7753199382595333, -4.751597592285044, -5.351900685535859, -5.538947192058466, -3.6838339356894148, -5.717756655548977, -5.9396873504379775, -6.193602560418941], [-4.999514188504179, -4.700587505853424, -4.67022803135639, -4.81576740495976, -4.92872993147181, -5.418517964945142, -5.4525477135314535, -5.471965799388554, -5.601583292691941, -5.294558257397029, -2.0675057720915833, -6.606510097282728, -6.66295140818768, -6.268297216183731, -1.8472066532219233, -6.801101746668497, -6.464629510047284, -5.518112402024337, -2.6153829107633095, -5.40926720517299, -7.042937430994859, -6.122880216325227, -4.984950824316282, -6.978988706394586, -2.5341534752438712, -6.335738496979263, -7.22903971062872, -5.047082605423289, -6.606510097282728, -7.195517018590076, -3.3881221963080814, -6.764734102497622, -6.935691900641261, -6.06802185577318, -5.518112402024337, -6.21743879895024, -3.306166689176508, -4.5835098665078435, -4.096018906794089, -2.7778687007936327, -3.149770331333854, -4.522505810044474, -3.6065186143693406, -4.579485716208118, -3.9009018998360694, -5.695109350702764, -4.592420415540891, -4.39850374434322, -4.858371250530563, -4.687974726037726, -4.284600731462279, -4.828050056815601, -5.7714823294873385, -4.760362534007699, -3.6910492160852075, -3.1337486268025883, -5.114702793098268, -5.044521784561615, -4.990993138772245, -5.21262268525524, -5.1495981689488834, -4.939033399841534], [-5.573013579156251, -4.7557456323553735, -4.835888625843566, -5.4310032481645445, -5.375816832597254, -5.748275644237248, -5.617589203744957, -5.426286258286405, -5.527732874623095, -6.027336973645122, -5.771265162461947, -3.4363190171520683, -4.863649827007062, -5.342810536129083, -3.9603112459777443, -3.1741279621323484, -6.522548369609561, -4.089429329592385, -5.667234434234122, -6.573100648772392, -5.74180312973163, -3.6814950844272993, -4.555731535734934, -2.60141208694855, -6.057840427938536, -2.6538446056843625, -7.3352407008192895, -2.4633202784204524, -3.0508881488790776, -5.069846636151324, -3.6495408996227328, -4.548858656447172, -5.7613477258046, -6.031638055544512, -6.61856302284915, -5.764642621701453, -4.906092884516815, -3.945751808422034, -4.075923906392578, -4.145352412824341, -4.3284585910787134, -4.1375398730875474, -4.386633312115669, -4.864991208831262, -5.2459274106602996, -5.389330551763976, -4.366047982499911, -3.5236520927129136, -3.6216686341149815, -3.0467401241562313, -4.4457511661828155, -3.28056139498962, -5.728982441302569, -3.2282776746379773, -3.3483163872180257, -3.6120365041642786, -4.280940073114231, -4.240597235287379, -4.439591885822281, -4.571620648562269, -5.13801612348307, -5.583972592945972], [-5.648871675811755, -4.741030782968909, -5.070478143556586, -5.038332518583169, -4.984450311189462, -5.652113169735926, -5.878916005474925, -6.018576120040821, -5.698633185370818, -5.924757364823166, -1.8468585970227798, -6.844251516414859, -7.126600242891277, -6.081398705962876, -2.934905888757429, -3.6595354362414967, -6.817864761241664, -3.5921374614243615, -2.9623018840998165, -7.15537920744132, -5.837035508229937, -2.956367800752568, -5.44072677540776, -6.135188880549014, -2.35874447462609, -6.817864761241664, -7.744626792983114, -1.829808862763535, -4.656579250518651, -7.012765100242478, -3.346772137573546, -6.156466278996299, -6.772055225210369, -5.459294948136628, -6.2464145156592386, -6.772055225210369, -3.9054574859103, -5.259720143195114, -3.484916480113615, -4.59888775210554, -4.26703708287839, -5.280773552392946, -4.568219053541838, -5.236954932422101, -4.227597595795837, -6.429913235626207, -5.14878310004672, -4.299244656170703, -5.066854951187166, -5.722730736949879, -4.343868104023761, -4.895290258966649, -5.924757364823166, -4.2366225736701635, -3.439140235431567, -5.290903070631802, -4.913738130330258, -5.32777360644013, -5.700335313441349, -4.118883335232663, -6.285399962372192, -5.825384891009962], [-4.134798672989471, -4.089847285127205, -4.1376763728170864, -4.175860159787245, -3.5867437074551245, -4.073498147125676, -4.247208240584712, -4.485742942219219, -4.161001045383496, -4.76664532768562, -4.782994465687151, -5.224827217966189, -5.79734641073752, -5.371880635922686, -5.362028339479674, -4.805217602471861, -5.696541711615555, -5.1428140663053545, -4.243997964954463, -5.696541711615555, -6.074978147335799, -4.388579193765571, -4.887134724939747, -5.433124261163405, -5.127188748402274, -5.465212575714905, -5.5681605449673475, -3.871109027280912, -4.3522115495946965, -4.887134724939747, -1.4812734394569864, -4.9697213159490214, -5.42265296129611, -5.5681605449673475, -5.696541711615555, -5.592551998091507, -3.9025017396887742, -4.303421385425264, -4.377529357578986, -3.2036759521599856, -4.414847120586181, -4.370230055097374, -4.283218678107745, -4.293269013961246, -4.570900750559526, -4.639893622046477, -4.473572406598963, -3.944368319081564, -3.9659778034144195, -4.663991173625537, -3.393956618621509, -4.489832927470744, -3.9232159440763374, -4.221810049979101, -3.677082874537429, -4.095356940938174, -3.0285527594662907, -4.570900750559526, -4.348646483430199, -4.449666885745409, -4.481669616831582, -4.21556002963393], [-5.468543074118657, -4.3409301123745765, -4.939132602788878, -5.030955774234888, -5.486636817561356, -5.444018684255573, -5.476853371252285, -5.808607963491769, -5.908385471304435, -5.666153323427702, -2.5849374499856705, -5.739356727450997, -6.452595908093631, -6.391754248837386, -1.3630653142736813, -7.218432822021854, -6.741508749931544, -4.321907198551311, -2.3295168721177877, -7.554905058643067, -7.600367432719824, -7.660265574300893, -6.068262723688894, -7.064282142194595, -2.4089621725210324, -6.261701120589188, -7.218432822021854, -7.400754378815808, -6.405740490812126, -6.66108718262097, -3.202908170420881, -7.345184527660997, -7.318516280578836, -7.17951740577218, -6.2346317986209705, -7.685266876506311, -3.523815890500982, -5.001129001779655, -4.254551658580372, -3.1791480369827805, -3.272238460048792, -4.979751531007288, -4.696520976240299, -4.996816791561475, -3.7821441205484283, -6.068262723688894, -4.767838317222632, -4.961310103104565, -3.3169412686460493, -4.254142074302682, -3.686717843919221, -4.781622132973809, -6.395232513213711, -4.882290151848487, -4.035101934573064, -4.142657840794326, -4.886134831709296, -5.154991621688459, -5.261248596393322, -5.763145589415012, -5.307128202144016, -5.4187681732866855], [-3.7153348719791315, -4.23835637018879, -5.401341822408315, -5.655381048015406, -5.3230099791384085, -5.723838105173554, -5.787159933132683, -6.191069830974041, -6.118044695959151, -6.512733837722823, -2.85919837658876, -7.902248813542757, -2.4061387894065027, -6.693770047296114, -2.250325724429897, -7.350962550860164, -6.005451461342365, -3.2057646919026945, -3.1269276063809657, -8.282234836349936, -4.929356031958588, -4.893377430923357, -4.350779231381914, -5.039126698093967, -3.0950598675329433, -2.702149075140166, -7.065140944061303, -6.4656293685353505, -6.6671525062961585, -1.5532677412410396, -3.407685858090993, -6.917395410231305, -4.818860209735889, -8.330553413620743, -4.697144805661785, -7.44034239452964, -4.201345166462018, -5.036551747250591, -4.447052124631202, -4.271306064673767, -4.3875307527168115, -5.68024420720805, -5.463555006909888, -4.6680077696105, -4.729585831829687, -6.877248342217893, -5.7894972929813875, -4.744562286019245, -4.863271676535759, -5.996776708148691, -5.115645847928164, -4.782587554878788, -6.69569868820252, -5.6198664278662305, -4.372289459396023, -3.8304137647360905, -5.143464614214886, -5.692911512408815, -5.442384299786921, -6.144390895990383, -6.553756817069401, -6.771887826560066], [-5.439192275092616, -4.781176186054329, -4.766861598657258, -5.578787193555129, -5.543563426336324, -5.577507602849504, -5.742587353208953, -6.240425781448133, -6.11031213333427, -6.461710020172159, -2.6665339561656087, -5.7471225083743445, -6.597455272456702, -6.655866034613116, -2.1677366785812078, -7.063212610820987, -6.615344837207477, -1.8542823223629723, -2.8203019407701473, -7.921874229858505, -5.964600322152877, -7.4190807777897785, -7.115398363991557, -7.50316389500032, -2.434370037607787, -6.75042461723813, -8.501692725111447, -2.436966652419961, -5.508344242912733, -7.262628600974241, -3.3318853658660776, -6.320468489121669, -4.662752576421778, -7.96269622437876, -4.3951216252530045, -7.773454224740232, -3.4248833433841024, -4.6171877485771935, -4.856615893655904, -5.148452521314832, -3.262215858586344, -5.067705520626301, -4.995134827791466, -4.808655260848847, -4.056803079326151, -6.372894286461519, -5.283769734998518, -4.869131701587735, -4.356292992089062, -5.580068423711178, -4.205055407055328, -5.300108397353307, -6.413362236014366, -4.081705328198343, -3.5651654031689133, -4.4607321960690065, -4.914213385439241, -3.3562768189902368, -5.301077859377855, -5.2191716350828905, -5.66568217643417, -5.148452521314832], [-4.692472771166889, -4.3008934499719516, -4.549181741142988, -4.955265087292482, -5.083773336506403, -5.36724487642184, -5.286555965171697, -5.606625988867734, -5.3561026998685985, -5.6766361544403825, -6.310506725070788, -5.345083304618988, -6.354631529979725, -5.436838168221035, -2.778870257014558, -5.888394383529467, -5.518132910614282, -3.971194340356585, -5.541198183545278, -7.353160360090853, -5.418980550821029, -3.590537431731367, -3.3241709789128793, -1.8944436557320132, -5.91226186493611, -3.390599447549114, -7.2559966116372046, -3.6807421830309885, -3.742794171598555, -5.40143624117012, -6.914247317915148, -6.268246915780905, -5.966865998970962, -6.802329401711162, -7.18453764765506, -6.063197107909394, -4.040088147996806, -3.4233375319614976, -2.7095546985241823, -4.374927246403353, -3.562959147374605, -4.404434128912793, -4.579296940419071, -4.391681565015401, -4.438151297809303, -5.606625988867734, -4.3282656378478706, -4.1449534915722195, -4.0950638220693705, -3.3913758451036427, -5.0164696546102965, -4.409796072054178, -6.030044900592494, -3.187603169643488, -2.500811269450856, -3.4121731416766146, -5.192891489277063, -4.804034117568558, -4.273312053475689, -5.056716535118889, -5.514880875227905, -5.219114684376165], [-5.262101982971463, -4.751704716461968, -4.4750082149096455, -5.006514043913175, -5.07515161250067, -5.251452255054804, -5.285938431125974, -5.392274080942006, -5.414469813333791, -5.840086736879263, -2.606359904105076, -6.823463761969788, -6.89757173412351, -5.234645136738424, -1.8016665052081617, -5.931967688708849, -5.7146991018376605, -5.437169400849897, -1.672652532530341, -6.72630001351614, -6.773702252410724, -5.308112288620296, -6.00575385876808, -6.6810434219280195, -2.3521748840468923, -5.96169386897405, -7.419447194076086, -6.095225261598572, -5.008173795331539, -6.397795946544104, -5.514028406375509, -5.907173030095633, -6.070893160939042, -4.572171963363293, -6.91979487090822, -6.823463761969788, -3.666029218824578, -4.169934399745995, -4.315857524665681, -2.6071122929751187, -3.822642418908326, -5.326212330263914, -4.3600448319184775, -4.917707706137853, -3.8075287810982776, -5.953110125282659, -5.187841070540621, -4.792366055507543, -4.8403533292283845, -5.164264339976624, -4.320022461964966, -3.8952676954062846, -5.745470760504414, -4.036941945426083, -4.228361366337563, -4.708930900478682, -5.247223918945283, -5.006514043913175, -3.8952676954062846, -4.574322501826521, -5.886970322778113, -5.748949024880739], [-5.49714261021104, -4.668772759709356, -4.7793028170607235, -5.269011517385791, -5.403728165571614, -5.600190986110487, -5.607143505425369, -5.736034518493389, -5.749350294469161, -5.980569259788917, -1.8463761496649909, -6.7957313909118255, -6.842614976810675, -6.277301167760615, -1.6016725181239624, -6.834646807161499, -6.483542908272222, -3.3781298559126016, -1.9147238137262874, -7.264209466848723, -6.97044834832056, -6.2503937148406905, -6.323821183395508, -6.466922027036182, -2.304012035094697, -6.232849405189781, -7.182292344380837, -4.500040235600472, -6.194465162181466, -7.045955900828706, -3.7335540179496816, -6.633976111699347, -6.224191342446667, -7.560054849939665, -6.541029999189903, -7.7391030813886506, -3.9953827115751284, -5.225208894727398, -4.2623981472183505, -4.983820106246462, -3.5900722944719905, -4.880831478599072, -4.625152137233465, -5.350160419862394, -4.464551422640724, -6.03257404477923, -5.25911044640308, -4.290672925686517, -4.802954305142684, -5.287422079228971, -4.798800612773991, -4.757191363453292, -5.977196575310277, -4.844383490556383, -2.946883968159507, -4.887603513509017, -5.795997983260946, -5.6188395451885595, -4.719156549714065, -5.921535728204724, -6.1778672707724285, -5.694577804815043], [-4.469097220436953, -3.6430767584970343, -3.7844067498952314, -3.931680914923784, -4.272838155799217, -4.129621978555971, -4.297357772973536, -4.384369149963165, -4.338992745625113, -4.178787025578902, -4.976420213651742, -4.292052720743842, -5.461928029433443, -5.887479896108345, -4.57562801124297, -5.526466550571014, -6.262964172004573, -4.4462034005711, -5.238784478119233, -6.017291507630335, -6.355745905455539, -5.968972930359527, -5.520424236115051, -6.328346931267425, -5.70515833931439, -4.218953067304238, -6.6434279779073195, -5.648805402763258, -5.5635078222513625, -4.870360641691417, -6.412904319295488, -1.949196049230905, -5.968972930359527, -4.4990139814749455, -6.027241838483503, -6.442757282445169, -4.057336183871974, -4.448263133534111, -4.000410247075965, -3.7940068236242506, -4.150834895195163, -4.166267300233975, -4.566303934367847, -4.3901887590164295, -4.028468199871122, -4.376661982718227, -4.915384323065373, -3.014652447863089, -3.369082406994758, -4.61381880997536, -4.55706599338291, -3.5814225774035497, -4.577972678202224, -4.001728638829222, -3.270401473211861, -3.264703452097223, -3.9718368746319084, -3.132988733879366, -4.361423592613295, -2.595191993338583, -4.516519898988561, -4.912100247864183], [-3.9699474093476774, -4.189042372135887, -4.058989243887689, -4.102030995146257, -4.171520021443684, -4.318254103615893, -4.424253392850214, -4.758810531535327, -4.504029895277935, -4.414714369803455, -3.0879080140350674, -5.469976217597951, -3.790334046490602, -5.366435538657111, -3.235086137375257, -5.727805326900051, -5.408282648592611, -4.971959552124542, -5.070590155566169, -5.076706382583605, -6.181472536826093, -5.921961341341008, -4.9999725883522155, -5.5070174892783, -1.5568628976878147, -5.5070174892783, -6.110013572843948, -4.0678979887767985, -5.089052218405905, -5.200643283814367, -3.6661684098537215, -4.557328477002297, -5.479108701161223, -5.159821289294111, -5.479108701161223, -5.616579691789827, -3.7152580200502454, -4.23556238777078, -4.212031890360586, -3.866007780968386, -4.0678979887767985, -4.4633463186414915, -4.142178063159669, -4.318254103615893, -4.276053749125516, -4.772293881872614, -4.332726136224427, -4.11133338780857, -3.9089648990693253, -4.309670359924501, -3.9127889955077286, -4.338574106106851, -4.528549512452254, -4.149433234040841, -3.5292744136497762, -3.6009751629760713, -4.350373653038005, -4.828122331425556, -4.047963773875981, -4.665125047458004, -4.673288358097166, -4.633122316371831], [-4.806719860900339, -3.6507473420444714, -4.429814962058902, -4.347330952102998, -4.3949170222076255, -4.698366552912107, -4.809448375553543, -4.35423944244681, -4.534137594904308, -4.3611959922401695, -2.871768725634745, -4.598399529122678, -5.564587232141456, -5.519124858064699, -2.4250566837898457, -5.688201188108633, -5.967149580591659, -5.01834957015221, -2.505498094634157, -5.3939617151106924, -6.3813483686685775, -5.360223575478843, -5.821732580733155, -5.742268409378909, -2.5427453130265225, -6.0299504818306895, -5.949907774157153, -6.048642614842842, -6.106911522966818, -6.522426966928483, -1.9368412502165815, -5.384205540165328, -3.2469989725553616, -5.166904264475346, -4.662348253722951, -5.93295821584338, -4.052787789271272, -3.8964417188805776, -4.725629703670472, -3.994277013931218, -3.570836586604418, -4.883136091344702, -4.20182336843176, -4.894970548991704, -4.1342203512338624, -5.159122124033291, -4.442985425248647, -4.740819869164446, -4.583031498894364, -4.589588899440523, -4.4058064220068935, -4.87144005158151, -4.8541485544714496, -4.806719860900339, -4.241282205172307, -4.937530163410501, -4.382360803432213, -5.0116381355642226, -4.368201274828579, -4.782492565565015, -4.544565218066568, -4.4563316672015425]],
probHashThreshold: 0.015,
probHashChars: {'1': 1, '0': 0, '3': 3, '2': 2, '5': 5, '4': 4, '7': 7, '6': 6, '9': 9, '8': 8, 'A': 36, 'C': 38, 'B': 37, 'E': 40, 'D': 39, 'G': 42, 'F': 41, 'I': 44, 'H': 43, 'K': 46, 'J': 45, 'M': 48, 'L': 47, 'O': 50, 'N': 49, 'Q': 52, 'P': 51, 'S': 54, 'R': 53, 'U': 56, 'T': 55, 'W': 58, 'V': 57, 'Y': 60, 'X': 59, 'Z': 61, 'a': 10, 'c': 12, 'b': 11, 'e': 14, 'd': 13, 'g': 16, 'f': 15, 'i': 18, 'h': 17, 'k': 20, 'j': 19, 'm': 22, 'l': 21, 'o': 24, 'n': 23, 'q': 26, 'p': 25, 's': 28, 'r': 27, 'u': 30, 't': 29, 'w': 32, 'v': 31, 'y': 34, 'x': 33, 'z': 35},
userTransitions: {
search: {}
},
searchEngines: [], //Variable for content extraction fw.
rArray: [], //Variable for content extraction fw.
extractRules: {}, //Variable for content extraction fw.
payloads: {}, //Variable for content extraction fw.
anonSearchEngines: [], //Variable for content extraction fw.
anonRArray: [], //Variable for content extraction fw.
anonExtractRules: {}, //Variable for content extraction fw.
anonPayloads: {}, //Variable for content extraction fw.
messageTemplate: {},
anonIdMappings: {},
patternsURL: 'https://cdn.cliqz.com/human-web/patterns',
anonPatternsURL: 'https://cdn.cliqz.com/human-web/patterns-anon',
configURL: 'https://safe-browsing-proxy-network.cliqz.com/config',
searchCache: {},
ts : "",
mRefresh : {},
can_url_match :{},
ismRefresh : false,
activityDistributor : Components.classes["@mozilla.org/network/http-activity-distributor;1"]
.getService(Components.interfaces.nsIHttpActivityDistributor),
userTransitionsSearchSession: 5*60,
key: ["source","protocol","authority","userInfo","user","password","host","port","relative","path","directory","file","query","anchor"],
q: {
name: "queryKey",
parser: /(?:^|&)([^&=]*)=?([^&]*)/g
},
parser: {
strict: /^(?:([^:\/?#]+):)?(?:\/\/((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?))?((((?:[^?#\/]*\/)*)([^?#]*))(?:\?([^#]*))?(?:#(.*))?)/,
loose: /^(?:(?![^:@]+:[^:@\/]*@)([^:\/?#.]+):)?(?:\/\/)?((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?)(((\/(?:[^?#](?![^?#\/]*\.[^?#\/.]+(?:[?#]|$)))*\/?)?([^?#\/]*))(?:\?([^#]*))?(?:#(.*))?)/
},
activeUsage : 0,
activeUsageThreshold : 2,
pageStatusCountSent: null,
actionStats: null,
actionStatsLastSent: null,
bloomFilter: null,
bf:null,
strictQueries:[],
oc: null,
SAFE_QUORUM_ENDPOINT: "https://safe-browsing-quorum.cliqz.com/",
SAFE_QUORUM_PROVIDER: "https://safe-browsing-quorum.cliqz.com/config",
quorumBloomFilters: {},
_md5: function(str) {
return md5(str);
},
maskURL: function(url){
var url_parts = null;
var masked_url = null;
// Fix
// url_parts = CliqzHumanWeb.parseUri(url);
url_parts = CliqzHumanWeb.parseURL(url);
// TO BE FIXED
if (!url_parts) return '';
// Fix
if (CliqzHumanWeb.dropLongURL(url)) {
//Explicit check for google search url.
if(url_parts['hostname'].indexOf('google') > 0){
if(url_parts['query_string']){
var query_stringsParts = CliqzHumanWeb.parseQueryString(url_parts['query_string']);
if(query_stringsParts['url']){
masked_url = query_stringsParts['url'][0];
masked_url = CliqzHumanWeb.maskURL(decodeURIComponent(''+masked_url));
return masked_url;
}
}
}
masked_url = url_parts.protocol + "://" + url_parts.hostname + "/ (PROTECTED)" ;
return masked_url;
}
return url;
},
isShortenerURL: function(url) {
try {
var url_parts = CliqzHumanWeb.parseURL(url);
if (!url_parts) return true;
if ((url_parts.hostname.length < 8) && (url_parts.path.length > 4)) {
var v = url_parts.path.split('/')
for(var i=0;i<v.length;i++) if (CliqzHumanWeb.isHash(v[i])) return true;
}
return false;
} catch(ee) {
return true;
}
},
getTime:function() {
try { var ts = CliqzUtils.getPref('config_ts', null)} catch(ee){};
if(!ts){
var d = null;
var m = null;
var y = null;
var h = null;
var hr = null;
var _ts = null;
d = (new Date().getDate() < 10 ? "0" : "" ) + new Date().getDate();
m = (new Date().getMonth() < 9 ? "0" : "" ) + parseInt((new Date().getMonth()) + 1);
h = (new Date().getUTCHours() < 10 ? "0" : "" ) + new Date().getUTCHours();
y = new Date().getFullYear();
_ts = y + "" + m + "" + d + "" + h;
}
else{
h = (new Date().getUTCHours() < 10 ? "0" : "" ) + new Date().getUTCHours();
_ts = ts + "" + h;
}
return _ts;
},
isSuspiciousURL: function(aURI) {
try {
var url_parts = {};
url_parts = CliqzHumanWeb.parseURL(aURI);
if (!url_parts) return true;
_log(JSON.stringify(url_parts));
if (aURI.indexOf('about:') == 0) return true;
if (/^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/.test(url_parts.hostname)) {
return true;
}
if (url_parts.username!="" || url_parts.password!="" ) {
return true;
}
if (url_parts.port != "" & (url_parts.port!="80" && url_parts.port!="443")) {
return true;
}
if ( url_parts.protocol != "http" & url_parts.protocol!="https" & url_parts.protocol != "") {
return true;
}
var pos_hash_char = aURI.indexOf('#');
if (pos_hash_char > -1) {
if (CliqzHumanWeb.checkSearchURL(aURI) == -1 && (aURI.length - pos_hash_char) >= 10) {
_log("Dropped because of # in url: " + decodeURIComponent(aURI));
return true;
}
}
if ( url_parts.hostname.indexOf('localhost') > -1){
return true;
}
_log("Sanitize: URL is ok: " + aURI);
return false;
} catch(ee) {
// if there were any exception, we return true for safety
_log("Exception in isSuspiciousURL: " + ee);
return true;
}
},
calculateStrictness: function(url, page_doc) {
var strict_value = true;
if (page_doc && page_doc['x'] && page_doc['x']['canonical_url']) {
// there is canonical,
var can_url_parts = CliqzHumanWeb.parseURL(page_doc['x']['canonical_url']);
var url_parts = CliqzHumanWeb.parseURL(url);
if (!url_parts || !can_url_parts) return true;
if (url_parts.hostname!=null && url_parts.hostname!='' && url_parts.hostname==can_url_parts.hostname) {
// both canonical and url have a hostname and is the same,
if ((page_doc['x']['canonical_url'] != url) && (page_doc['x']['canonical_url'].length < url.length)) {
// the page has a canonical of same domain, which usually is a sign that is public,
// and the canonical is not the same url, which comes out of automatic generation
// of canonicals
strict_value = false;
}
}
}
return strict_value;
},
dropLongURL: function(url, options) {
try {
if (options==null) options = {'strict': false};
if (CliqzHumanWeb.checkForEmail(url)) return true;
var url_parts = CliqzHumanWeb.parseURL(url);
if (!url_parts) return true;
if (options.strict == true) {
if (url_parts.query_string && url_parts.query_string.length > CliqzHumanWeb.qs_len*0.75) return true;
// check the number of parameters too,
if (url_parts.query_string) {
var v = url_parts.query_string.split(/[&;]/);
if (v.length > 1) {
// that means that there is a least one &; hence 2 params
return true;
}
for (var i=0; i<v.length; i++) {
if (v[i].length>3 && CliqzHumanWeb.isHash(v[i])) return true;
}
if (CliqzHumanWeb.checkForLongNumber(url_parts.query_string, 8)!=null) return true;
}
if (CliqzHumanWeb.checkForLongNumber(url_parts.path, 8)!=null) return true;
}
else {
if (url_parts.query_string && url_parts.query_string.length > CliqzHumanWeb.qs_len) return true;
if (url_parts.query_string) {
var v = url_parts.query_string.split(/[&;]/);
if (v.length > 4) {
// that means that there is a least one &; hence 5 params
return true;
}
if (CliqzHumanWeb.checkForLongNumber(url_parts.query_string, 12)!=null) return true;
}
if (CliqzHumanWeb.checkForLongNumber(url_parts.path, 12)!=null) return true;
}
var vpath = url_parts.path.split(/[\/\._ \-:\+;]/);
for (var i=0; i<vpath.length; i++) {
if (vpath[i].length > CliqzHumanWeb.rel_part_len) {
return true;
}
if (options.strict == true) {
// if strict, check the no token in path looks like a hash
if (vpath[i].length>5 && CliqzHumanWeb.isHash(vpath[i])) return true;
}
else {
if (vpath[i].length>12 && CliqzHumanWeb.isHash(vpath[i])) return true;
}
}
var vpath = url_parts.path.split('/');
for (var i=0; i<vpath.length; i++) {
var cstr = vpath[i].replace(/[^A-Za-z0-9]/g,'');
var mult = 1.0;
if (options.strict == true) mult = 0.5;
if (cstr.length > CliqzHumanWeb.rel_segment_len * mult) {
if (CliqzHumanWeb.isHash(cstr)) return true;
}
}
var v = [/\/admin([\/\?#=]|$)/i, /\/wp-admin([\/\?#=]|$)/i,
/\/edit([\/\?#=]|$)/i, /[&\?#\/]share([\/\?#=]|$)/i,
/[&\?#\/;]sharing([\/\?#=]|$)/i, /[&\?#\/;]logout([\/\?#=]|$)/i,
/WebLogic/i, /[&\?#\/;]token([\/\?#=_;]|$)/i,
/[&\?#\/;]trk([\/\?#=_]|$)/i, /[&\?#\/=;](http|https)(:\/|\%3A\%2F)/];
// url_rel contains path and query_string
//
var url_rel = (url_parts.path || '/') + '?' + (url_parts.query_string || '');
for (var i=0;i<v.length;i++) if (v[i].test(url_rel)) return true;
// checks specific to the query string
//
// real query string (?) or a 'fake' one with a sharp on the path, they should be treated the same way
//
var path_query_string = null;
var ind_pos = url_parts.path.indexOf('#');
if (ind_pos!=-1) path_query_string = url_parts.path.slice(ind_pos, url_parts.path.length);
if ((url_parts.query_string && url_parts.query_string.length > 0) || (path_query_string && path_query_string.length > 0)) {
var v = [/[&\?#_\-;]user/i, /[&\?#_\-;]token/i, /[&\?#_\-;]auth/i, /[&\?#_\-;]uid/i, /[&\?#_\-;]email/i,
/[&\?#_\-;]usr/i, /[&\?#_\-;]pin/i, /[&\?#_\-;]pwd/i, /[&\?#_\-;]password/i, /[&\?#;]u[=#]/i, /[&\?#;]url[=#]/i,
/[&\?#_\-;]http/i, /[&\?#_\-;]ref[=#]/i, /[&\?#_\-;]red[=#]/i, /[&\?#_\-;]trk/i, /[&\?#_\-;]track/i,
/[&\?#_\-;]shar/i, /[&\?#_\-;]login/i, /[&\?#_\-;]logout/i, /[&\?#_\-;]session/i,
];
if (url_parts.query_string && url_parts.query_string.length > 0) {
for (var i=0; i<v.length; i++) if (v[i].test('?' + url_parts.query_string)) return true;
}
if (path_query_string && path_query_string.length > 0) {
for (var i=0; i<v.length; i++) if (v[i].test(path_query_string)) return true;
}
}
return false;
} catch(ee) {
// if there were any exception, we return true for safety
_log("Exception in dropLongURL: " + ee);
return true;
}
},
cleanDocCache: function() {
for(var key in CliqzHumanWeb.docCache) {
if ((CliqzHumanWeb.counter - CliqzHumanWeb.docCache[key]['time']) > 3600*CliqzHumanWeb.tmult) {
delete CliqzHumanWeb.docCache[key];
}
}
},
cleanHttpCache: function() {
for(var key in CliqzHumanWeb.httpCache) {
if ((CliqzHumanWeb.counter - CliqzHumanWeb.httpCache[key]['time']) > 60*CliqzHumanWeb.tmult) {
delete CliqzHumanWeb.httpCache[key];
}
}
for(var key in CliqzHumanWeb.httpCache401) {
if ((CliqzHumanWeb.counter - CliqzHumanWeb.httpCache401[key]['time']) > 60*CliqzHumanWeb.tmult) {
delete CliqzHumanWeb.httpCache401[key];
}
}
},
getHeaders: function(strData) {
var o = {};
o['status'] = strData.split(" ")[1];
var l = strData.split("\n");
for(var i=0;i<l.length;i++) {
if (l[i].indexOf('Location: ') == 0) {
o['loc'] = decodeURIComponent(l[i].split(" ")[1].trim());
}
if (l[i].indexOf('WWW-Authenticate: ') == 0) {
var tmp = l[i].split(" ");
var tmp = tmp.slice(1, tmp.length).join(" ");
o['auth'] = tmp;
}
}
return o;
},
httpObserver: {
// check the non 2xx page and report if this is one of the cliqz result
observeActivity: function(aHttpChannel, aActivityType, aActivitySubtype, aTimestamp, aExtraSizeData, aExtraStringData) {
try {
var aChannel = aHttpChannel.QueryInterface(nsIHttpChannel);
// Return if it's a private tab.
if(aChannel.isChannelPrivate != undefined && aChannel.isChannelPrivate) return;
var url = decodeURIComponent(aChannel.URI.spec);
var ho = CliqzHumanWeb.getHeaders(aExtraStringData);
var status = ho['status'];
var loc = ho['loc'];
var httpauth = ho['auth'];
if(gadurl.test(url) && aChannel && aChannel.referrer) {
var refU = aChannel.referrer.spec;
CliqzHumanWeb.linkCache[url] = {'s': ''+refU, 'time': CliqzHumanWeb.counter};
}
if (status=='301' || status == '302') {
CliqzHumanWeb.httpCache[url] = {'status': status, 'time': CliqzHumanWeb.counter, 'location': loc};
}
else if (status=='401') {
CliqzHumanWeb.httpCache401[url] = {'time': CliqzHumanWeb.counter};
}
else if(status){
CliqzHumanWeb.httpCache[url] = {'status': status, 'time': CliqzHumanWeb.counter};
}
} catch(ee){};
}
},
historyTimeFrame: function(callback) {
Cu.import('resource://gre/modules/PlacesUtils.jsm');
var history = [];
var min, max;
var res = [];
var st = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase).DBConnection.createStatement("SELECT min(last_visit_date) as min_date, max(last_visit_date) as max_date FROM moz_places");
var res = [];
st.executeAsync({
handleResult: function(aResultSet) {
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
res.push({"minDate": row.getResultByName("min_date"), "maxDate": row.getResultByName("max_date")});
}
},
handleError: function(aError) {
_log("SQL error: " + aError.message);
callback(true);
},
handleCompletion: function(aReason) {
if (aReason != Components.interfaces.mozIStorageStatementCallback.REASON_FINISHED) {
_log("SQL canceled or aborted");
callback(null);
}
else {
try {
min = parseInt(res[0]['minDate'] / 1000);
max = parseInt(res[0]['maxDate'] / 1000);
} catch (ex) {}
callback(min, max);
}
}
});
},
SQL: function(sql, onRow, callback, parameters) {
// temporary fix to avoid console logs if human web is disabled
// the history listner should be handled better if HW module is disabled
if(!CliqzHumanWeb.dbConn) return;
var st = CliqzHumanWeb.dbConn.createAsyncStatement(sql);
for(var key in parameters) {
st.params[key] = parameters[key];
}
CliqzHumanWeb._SQL(CliqzHumanWeb.dbConn, st, onRow, callback);
},
_SQL: function(dbConn, statement, onRow, callback) {
statement.executeAsync({
onRow: onRow,
callback: callback,
handleResult: function(aResultSet) {
var resultCount = 0;
for (let row = aResultSet.getNextRow(); row; row = aResultSet.getNextRow()) {
resultCount++;
if (this.onRow) {
this.onRow(statement.row);
}
}
if (this.callback) {
this.callback(resultCount);
}
},
handleError: function(aError) {
_log("Error (" + aError.result + "):" + aError.message);
if (this.callback) {
this.callback(0);
}
},
handleCompletion: function(aReason) {
// Always called when done
}
});
statement.finalize();
},
deleteVisit: function(url) {
CliqzHumanWeb.SQL("delete from usafe where url = :url", null, null, {
url: CliqzHumanWeb.escapeSQL(url)
});
},
deleteTimeFrame: function() {
CliqzHumanWeb.historyTimeFrame(function(min, max) {
CliqzHumanWeb.SQL("delete from usafe where last_visit < :min", null, null, {
min: min
});
CliqzHumanWeb.SQL("delete from usafe where last_visit > :max", null, null, {
max: max
});
});
},
clearHistory: function() {
CliqzHumanWeb.SQL("delete from usafe");
},
historyObserver: {
onBeginUpdateBatch: function() {},
onEndUpdateBatch: function() {
CliqzHumanWeb.deleteTimeFrame();
},
onVisit: function(aURI, aVisitID, aTime, aSessionID, aReferringID, aTransitionType) {},
onTitleChanged: function(aURI, aPageTitle) {},
onBeforeDeleteURI: function(aURI) {},
onDeleteURI: function(aURI) {
CliqzHumanWeb.deleteVisit(aURI.spec);
},
onClearHistory: function() {
CliqzHumanWeb.clearHistory();
},
onPageChanged: function(aURI, aWhat, aValue) {},
onDeleteVisits: function() {},
QueryInterface: XPCOMUtils.generateQI([Ci.nsINavHistoryObserver])
},
linkCache: {},
cleanLinkCache: function() {
for(var key in CliqzHumanWeb.linkCache) {
if ((CliqzHumanWeb.counter - CliqzHumanWeb.linkCache[key]['time']) > 30*CliqzHumanWeb.tmult) {
delete CliqzHumanWeb.linkCache[key];
}
}
},
getRedirects: function(url, _res) {
var res = _res || []
try{
for(var key in CliqzHumanWeb.httpCache) {
if(CliqzHumanWeb.httpCache[key]){
if (CliqzHumanWeb.httpCache[key]['location']!=null && (CliqzHumanWeb.httpCache[key]['status']=='301' || CliqzHumanWeb.httpCache[key]['status']=='302')) {
if (CliqzHumanWeb.httpCache[key]['location']==url || decodeURIComponent(CliqzHumanWeb.httpCache[key]['location']) == url) {
res.unshift(key)
CliqzHumanWeb.getRedirects(key, res);
}
}
}
}
}
catch(ee){};
return res;
},
userSearchTransition: function(rq){
// now let's manage the userTransitions.search
if (rq!=null) {
var source = rq['qurl'];
var query = rq['q'].replace(/\s+/g, " ").trim().toLowerCase();
if (source && query && query.length>0) {
// we have both the source and the query,
// let's see if we have done the query
if (CliqzHumanWeb.userTransitions['search'][query] == null) {
CliqzHumanWeb.userTransitions['search'][query] = {'time': CliqzHumanWeb.counter, 'data': []}
}
CliqzHumanWeb.userTransitions['search'][query]['data'].push([source, CliqzHumanWeb.counter - CliqzHumanWeb.userTransitions['search'][query]['time']]);
}
}
},
getParametersQS: function(url) {
var res = {};
var KeysValues = url.split(/[\?&]+/);
for (var i = 0; i < KeysValues.length; i++) {
var kv = KeysValues[i].split("=");
if (kv.length==2) res[kv[0]] = kv[1];
}
return res;
},
getEmbeddedURL: function(targetURL) {
var ihttps = targetURL.lastIndexOf('https://')
var ihttp = targetURL.lastIndexOf('http://')
if (ihttps>0 || ihttp>0) {
// contains either http or https not ont he query string, very suspicious
var parqs = CliqzHumanWeb.getParametersQS(targetURL);
if (parqs['url']) {
return decodeURIComponent(parqs['url']);
}
}
else return null;
},
auxIsAlive: function() {
return true;
},
auxGetPageData: function(url, page_data, original_url, onsuccess, onerror) {
var error_message = null;
var req = Components.classes['@mozilla.org/xmlextras/xmlhttprequest;1'].createInstance();
/*
We need a try catch block here, because there are some URLs which throw malformed URI error,
hence stalling the double fetch on the same row.
Such URLs should not be there at the first place, but in-case they are, we set them as private.
*/
try{
req.open('GET', url, true);
}catch(ee){
onerror(url, page_data, original_url, error_message);
return;
}
req.overrideMimeType('text/html');
req.channel.loadFlags |= Ci.nsIRequest.LOAD_ANONYMOUS;
//req.withCredentials = false;
//req.setRequestHeader("Authorization", "true");
req.onload = function(){
if (req.status != 200 && req.status != 0 /* local files */){
error_message = 'status not valid: ' + req.status;
_log("Error on doublefetch: " + error_message);
req.onerror();
}
else {
// there has been a redirect, we cannot guarantee that cookies were
// not sent, therefore fail and consider as private
if (req.responseURL != url) {
if (decodeURI(decodeURI(req.responseURL)) != decodeURI(decodeURI(url))) {
error_message = 'dangerous redirect';
_log("Error on doublefetch: " + error_message);
_log("DANGER: " + url + ' ' + req.responseURL);
req.onerror();
return;
}
}
var hiddenWindow = Services.appShell.hiddenDOMWindow;
var parser = new hiddenWindow.DOMParser();
var doc = parser.parseFromString(req.responseText, "text/html");
var x = CliqzHumanWeb.getPageData(url, doc);
CliqzHumanWeb.docCache[url] = {'time': CliqzHumanWeb.counter, 'doc': doc};
onsuccess(url, page_data, original_url, x);
}
}
req.onerror = function() {
onerror(url, page_data, original_url, error_message);
}
req.ontimeout = function() {
error_message = 'timeout';
_log("Error on doublefetch: " + error_message);
req.onerror();
}
req.timeout = 10000;
req.send(null);
},
auxIntersection: function(a, b) {
var ai=0, bi=0;
var result = new Array();
while( ai < a.length && bi < b.length ) {
if (a[ai] < b[bi] ){ ai++; }
else if (a[ai] > b[bi] ){ bi++; }
else {
result.push(a[ai]);
ai++;
bi++;
}
}
return result;
},
auxUnion: function(a, b) {
var h = {};
for (var i = a.length-1; i >= 0; -- i) h[a[i]] = a[i];
for (var i = b.length-1; i >= 0; -- i) h[b[i]] = b[i];
var res = [];
for (var k in h) {
if (h.hasOwnProperty(k)) res.push(h[k]);
}
return res;
},
validDoubleFetch: function(struct_bef, struct_aft, options) {
// compares the structure of the page when rendered in Firefox with the structure of
// the page after.
_log("xbef: " + JSON.stringify(struct_bef));
_log("xaft: " + JSON.stringify(struct_aft));
// Check if struct_bef or struct_aft is not null, in case anyone is then we mark it as private.
// if any of the titles is null (false), then decline (discard)
if (!(struct_bef && struct_aft)) {
_log("fovalidDoubleFetch: found an empty structure");
return false;
}
if (!(struct_bef['t'] && struct_aft['t'])) {
_log("fovalidDoubleFetch: found an empty title");
return false;
}
// if any of the two struct has a iall to false decline
if (!(struct_bef['iall'] && struct_aft['iall'])) {
_log("fovalidDoubleFetch: found a noindex");
return false;
}
// check that there are not different number of frames (or iframes) with an internal
// link on the two different loads (with and with session), if so, the frame (iframe)
// might contain a password field. Cannot afford to fetch all iframes on page. Only
// internal are considered, externals are likely to vary a lot due to advertisement.
//
/*
Adding key to check how many pages will we loose if frame check is turned on
if (struct_bef['nfsh']==null || struct_aft['nfsh']==null || struct_bef['nfsh']!=struct_aft['nfsh']) {
_log("fovalidDoubleFetch: number of internal frames does not match");
return false;
}
if (struct_bef['nifsh']==null || struct_aft['nifsh']==null || struct_bef['nifsh']!=struct_aft['nifsh']) {
_log("fovalidDoubleFetch: number of internal iframes does not match");
return false;
}
*/
if (struct_bef['canonical_url'] != struct_aft['canonical_url']) {
// if canonicals are different, in principle are different pages,
if (struct_aft['canonical_url']!=null && struct_aft['canonical_url'].length>0 && struct_bef['canonical_url']==null) {
// unless in the case that struct_aft (after) has a canonical different than null
// and struct_bef has no canonical,
}
else {
return false;
}
}
if (options.structure_strict==true) {
// if there is enough html length, do the ratio, if below or above 10% then very imbalance, discard
var length_html_ok = true;
var length_text_ok = true;
var ratio_lh = (struct_bef['lh'] || 0) / ((struct_bef['lh'] || 0) + (struct_aft['lh'] || 0));
if ((struct_bef['lh'] || 0) > 10*1024) {
var ratio_lh = (struct_bef['lh'] || 0) / ((struct_bef['lh'] || 0) + (struct_aft['lh'] || 0));
if (ratio_lh < 0.10 || ratio_lh > 0.90) {
_log("fovalidDoubleFetch: lh is not balanced");
length_html_ok = false;
}
}
// if there is enough html length, do the ratio, if below or above 10% then very imbalance, discard
var ratio_nl = (struct_bef['nl'] || 0) / ((struct_bef['nl'] || 0) + (struct_aft['nl'] || 0));
if ((struct_bef['lh'] || 0) > 30) {
var ratio_nl = (struct_bef['nl'] || 0) / ((struct_bef['nl'] || 0) + (struct_aft['nl'] || 0));
if (ratio_nl < 0.10 || ratio_nl > 0.90) {
_log("fovalidDoubleFetch: nl is not balanced");
length_text_ok = false;
}
}
if (!length_text_ok && !length_html_ok) return false;
}
// check for passwords and forms if there is no canonical on the after (double fetched with no session)
if (struct_aft['canonical_url']==null || struct_aft['canonical_url']=='') {
// if had no password inputs before and it has after, decline
if ((struct_bef['nip'] == null || struct_aft['nip'] == null) || (struct_bef['nip'] == 0 && struct_aft['nip'] > 0)) {
_log("validDoubleFetch: fail nip");
return false;
}
// if had no forms before and it has after, decline
if ((struct_bef['nf'] == null || struct_aft['nf'] == null) || (struct_bef['nf'] == 0 && struct_aft['nf'] > 0)) {
_log("validDoubleFetch: fail text nf");
return false;
}
}
// compare that titles are equal, if not equal, use the jaccard coefficient, decline if <=0.5
var t1 = struct_bef['t'] || '';
var t2 = struct_aft['t'] || '';
var jc = 1.0;
if (t1!=t2) {
var vt1 = t1.split(' ').filter(function(el) {return el.length>1});
var vt2 = t2.split(' ').filter(function(el) {return el.length>1});
jc = CliqzHumanWeb.auxIntersection(vt1,vt2).length / CliqzHumanWeb.auxUnion(vt1,vt2).length;
if (Math.max(vt1.length, vt2.length) <= 4) {
// the longest titles is 4 tokens long, the, we are a bit flexible on title differences
if (jc >= 0.5) return true;
else {
_log("short title fail title overlap");
return false;
}
}
else {
// the longest titles has 4 or more tokens, be more restrictive
if (jc <= 0.5) {
// one last check, perhaps it's an encoding issue
var tt1 = t1.replace(/[^A-Za-z 0-9 \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~]*/g, '');
var tt2 = t2.replace(/[^A-Za-z 0-9 \.,\?""!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~]*/g, '');
if ((tt1.length > t1.length*0.5) && ((tt2.length > t2.length*0.5))) {
// if we have not decreased the titles by more than 50%
var vtt1 = tt1.split(' ').filter(function(el) {return el.length>1});
var vtt2 = tt2.split(' ').filter(function(el) {return el.length>1});
jc = CliqzHumanWeb.auxIntersection(vtt1,vtt2).length / CliqzHumanWeb.auxUnion(vtt1,vtt2).length;
// we are more demanding on the title overlap now
if (jc <= 0.80) {
_log("validDoubleFetch: fail title overlap after ascii");
return false;
}
}
else {
_log("validDoubleFetch: fail title overlap");
return false;
}
}
// if the titles are not a perfect match then check for more structural things like number of inputs
// that are type password and number of forms. This is prone to false positives because when not logged in
// legitimate sites something prompt you to register
// if had no password inputs before and it has after, decline
if ((struct_bef['nip'] == null || struct_aft['nip'] == null) || (struct_bef['nip'] == 0 && struct_aft['nip'] > 0)) {
_log("validDoubleFetch: fail nip");
return false;
}
// if had no forms before and it has after, decline
if ((struct_bef['nf'] == null || struct_aft['nf'] == null) || (struct_bef['nf'] == 0 && struct_aft['nf'] > 0)) {
_log("validDoubleFetch: fail text nf");
return false;
}
return true;
}
}
else {
// exactly same title
return true;
}
_log("validDoubleFetch: default option");
return false;
},
getCleanerURL: function(url) {
var clean_url = url;
// check first if there is a query string,
var url_parts = CliqzHumanWeb.parseURL(url);
if (!url_parts) return null;
if (url_parts && url_parts.query_string && url_parts.query_string != '') {
// it has a query string, either by ? # or ;
clean_url = url_parts.protocol + '://' + url_parts.hostname + url_parts.path;
}
else {
// it has neither query_string or hash or semicolon, so let's try to remove the last segment
if (url_parts && url_parts.path && url_parts.path != '') {
var qs = url_parts.path.split('/')
var cqs = [];
for (let i=0;i<qs.length;i++) if (qs[i]!='') cqs.push(qs[i]);
if (cqs.length >= 3) {
// more than 3, /a/b/c, let's remove the last one
var new_path = cqs.slice(0, cqs.length-1).join('/');
clean_url = url_parts.protocol + '://' + url_parts.hostname + '/' + new_path + '/';
}
}
}
if (clean_url != url) {
// they are different, sanity checks
if (CliqzHumanWeb.isSuspiciousURL(clean_url) || CliqzHumanWeb.dropLongURL(clean_url)) return url;
else return clean_url;
}
else return url;
},
fetchReferral: function(referral_url, callback) {
_log("PPP in fetchReferral: " + referral_url);
if (referral_url && referral_url!='') {
if (CliqzHumanWeb.docCache[referral_url]==null) {
CliqzHumanWeb.auxGetPageData(referral_url, null, null, function(referral_url) {
_log("PPP in fetchReferral success auxGetPageData: " + referral_url);
callback();
},
function(referral_url) {
_log("PPP in fetchReferral failure auxGetPageData: " + referral_url);
callback();
});
}
else {
_log("PPP in fetchReferral already in docCache: " + referral_url);
callback();
}
}
else callback();
},
doubleFetch: function(url, page_doc) {
// Need to add check for page MU.
// one last validation whether should be fetchable or not. If we cannot send that URL because it's
// private/suspicious/search_result page/etc. we can mark it as private directly
var isok = true;
if (page_doc ==null || page_doc['x'] == null) {
// this should not happen, but it does. Need to debug why the 'x' field gets lost
// right now, let's set is a private to avoid any risk
//
isok = false
}
if (page_doc && page_doc['x'] && page_doc['x']['iall'] == false) {
// the url is marked as noindex
isok = false;
}
// the url is suspicious, this should never be the case here but better safe
//
if (CliqzHumanWeb.isSuspiciousURL(url) == true) isok = false;
if (CliqzHumanWeb.dropLongURL(url)) {
_log("The url: " + url + " is long");
if (page_doc && page_doc['x'] && page_doc['x']['canonical_url']) {
// the url is to be drop, but it has a canonical URL so it should be public
if (CliqzHumanWeb.dropLongURL(page_doc['x']['canonical_url'])) {
// wops, the canonical is also bad, therefore mark as private
_log("The canonical url: " + page_doc['x']['canonical_url'] + " is also long");
isok = false;
}
else {
// there we are in the good scenario in which canonical looks ok although
// url did not
isok = true;
}
}
else {
if(page_doc['isMU']){
isok = true;
}
else{
isok = false;
}
}
}
if (isok) {
_log("going to double-fetch: " + url);
CliqzHumanWeb.auxGetPageData(url, page_doc, url, function(url, page_doc, original_url, data) {
// data contains the public data of the url double-fetch,
_log("success on doubleFetch, need further validation" + url);
if (CliqzHumanWeb.validDoubleFetch(page_doc['x'], data, {'structure_strict': false})) {
//
// If the double fetch is validated, we will now check for the iFrame and frameSets
// Since we only need the telemetry for now, this seems to be the right place
// we need to inject the page structure. The final event will have an extra key
// nifshmatch : true / false , nfshmatch: true / false.,
// nifshbf : Iframe count in struct_bef, nifshbf : framsetcount in before.
// This key is added to both page_doc and data.
//
let nifshmatch = CliqzHumanWeb.validFrameCount(page_doc['x'], data);
let nfshmatch = CliqzHumanWeb.validFrameSetCount(page_doc['x'], data);
data.nifshmatch = nifshmatch;
data.nfshmatch = nfshmatch;
data.nifshbf = page_doc.x.nifsh;
data.nfshbf = page_doc.x.nifsh;
//
// url, we should have the data of the double for the referral in CliqzHumanWeb.docCache
//
CliqzHumanWeb.fetchReferral(page_doc['ref'], function() {
var strict_value = CliqzHumanWeb.calculateStrictness(url, page_doc);
if (page_doc['ref'] && page_doc['ref']!='') {
// the page has a referral
_log("PPP: page has a referral, " + url + " < " + page_doc['ref']);
var hasurl = CliqzHumanWeb.hasURL(page_doc['ref'], url);
_log("PPP: page has a referral, " + url + " < " + page_doc['ref'] + ">>>> " + hasurl);
// overwrite strict value because the link exists on a public fetchable page
if (hasurl) strict_value = false;
}
else {
// page has no referral
_log("PPP: page has NO referral, " + url);
// we do not know the origin of the page, run the dropLongURL strict version, if
// there is no canonical or if there is canonical and is the same as the url,
}
_log("strict URL:" + url + " > " + strict_value);
if (CliqzHumanWeb.validDoubleFetch(page_doc['x'], data, {'structure_strict': strict_value})) {
// we do not know the origin of the page, run the dropLongURL strict version
if (CliqzHumanWeb.dropLongURL(url, {'strict': strict_value})) {
if (page_doc && page_doc['x'] && page_doc['x']['canonical_url']) {
if (CliqzHumanWeb.dropLongURL(page_doc['x']['canonical_url'], {'strict': strict_value})) {
_log("doubleFetch failed on dropLongURL strict=true:" + url);
CliqzHumanWeb.setAsPrivate(url);
return;
}
}
else {
_log("doubleFetch failed on dropLongURL strict=true:" + url);
CliqzHumanWeb.setAsPrivate(url);
return;
}
}
}
else {
// the strict version of validDoubleFetch fails for a page with no referral,
// since we do not know the origin mark as private
_log("doubleFetch failed on structure_strict=true: " + url);
CliqzHumanWeb.setAsPrivate(url);
return;
}
_log("success on doubleFetch, need further validation");
//
// we need to modify the 'x' field of page_doc to substitute any structural information about
// the page content by the data coming from the doubleFetch (no session)
//
var first_url_double_fetched = url;
//
// we need to modify the 'x' field of page_doc to substitute any structural information about
// the page content by the data coming from the doubleFetch (no session)
//
// at this point we might have 3 different urls: url, page_doc['x']['canonical_url'] (either
// one of them has to have passed the dropLongURL test), and finally we also have data['canonical_url']
// (which has not passed the dropLongURL test). In principle, data['canonical_url']==page_doc['x']['canonical_url']
// but is not always the case, different calls can give different urls
if (data['canonical_url']!=null && data['canonical_url'] != '' && CliqzHumanWeb.dropLongURL(data['canonical_url'])==false) {
page_doc['url'] = data['canonical_url'];
page_doc['x'] = data;
}
else {
if (page_doc['x']['canonical_url']!=null && page_doc['x']['canonical_url'] != '' && CliqzHumanWeb.dropLongURL(page_doc['x']['canonical_url'])==false) {
page_doc['url'] = page_doc['x']['canonical_url'];
page_doc['x'] = data;
page_doc['x']['canonical_url'] = page_doc['url'];
}
else {
// there was no canonical either on page_doc['x'] or in data or it was droppable
if (CliqzHumanWeb.dropLongURL(url)==false) {
page_doc['url'] = url;
page_doc['x'] = data;
if (page_doc['x']['canonical_url']!=null && page_doc['x']['canonical_url'] != '') {
page_doc['x']['canonical_url'] = url;
}
}
else {
// this should not happen since it would be covered by the isok checks, but better safe,
CliqzHumanWeb.setAsPrivate(url);
return;
}
}
}
var clean_url = CliqzHumanWeb.getCleanerURL(page_doc['url']);
if (clean_url != page_doc['url']) {
// we have a candidate for a cleaner url (without query_string) or without the last segment
// of the path, we want to double fetch it and if successful, then, we could replace the url
// because it would be cleaner hence safer
//
_log("going to clean_url double-fetch: " + clean_url);
CliqzHumanWeb.auxGetPageData(clean_url, page_doc, first_url_double_fetched, function(url, page_doc, original_url, data) {
_log("success on clean_url doubleFetch, need further validation");
if (CliqzHumanWeb.validDoubleFetch(page_doc['x'], data, {'structure_strict': false})) {
// if it the second double fetch is valid, that means that the clean_url is (url parameter) is
// equivalent, so we can replace
page_doc['url'] = url;
page_doc['x'] = data;
CliqzHumanWeb.setAsPublic(original_url);
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});
}
else {
// the page with the clean_urls does not return the same, it can be two cases here, one is that
// the content is just totally different, in this case, we should send the page with the unclean_url,
// the other case is that the content is different but now we have passwords where we did not have before,
// in such a case, it's safer to assume that the fragments cleaned were identifiying a user, and the
// website is redirecting to the login page, in such a case, we should not send the page at all, in fact, we
// should mark it as private just to be sure,
if (CliqzHumanWeb.debug) {
_log("checking clean_url, page_doc: " + JSON.stringify(page_doc));
_log("checking clean_url, data: " + JSON.stringify(data));
}
if (page_doc['x']['nip'] < data['nip']) {
// the page with url_clean have more input password fields or more forms, this is dangerous,
_log("failure on checking the password and forms for clean_url: " + original_url + " set as private");
CliqzHumanWeb.setAsPrivate(original_url);
}
else {
// safe, here we will send the url before clean_url
CliqzHumanWeb.setAsPublic(original_url);
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});
}
}
},
function(url, page_doc, original_url, error_message) {
_log("failure on clean_url doubleFetch! " + "structure did not match");
// there was a failure, the clean_url does not go to the same place, therefore it's better
// not to replace
CliqzHumanWeb.setAsPublic(original_url);
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});
});
}
else {
CliqzHumanWeb.setAsPublic(original_url);
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'page', 'payload': page_doc});
}
});
}
else {
_log("failure on doubleFetch! " + "structure did not match");
CliqzHumanWeb.setAsPrivate(url);
}
},
function(url, page_doc, original_url, error_message) {
_log("failure on doubleFetch! " + error_message);
CliqzHumanWeb.setAsPrivate(url);
});
}
else {
_log("doubleFetch refused to process this url: " + url);
CliqzHumanWeb.setAsPrivate(url);
}
},
hasURL: function(source_url, target_url) {
// the target_url is in the source_url
try {
var tt = CliqzHumanWeb.docCache[source_url];
if (tt) {
var cd = tt['doc'];
if (!cd) {
// fetch the content of the source_url,
//
_log("hasURL no CD!!! ");
return false;
}
}
else return false;
var target_url_no_protocol = target_url.replace(/^http(s?)\:\/\//, '');
var target_url_relative = null;
try {
var source_hostname = CliqzHumanWeb.parseURL(source_url).hostname;
var target_hostname = CliqzHumanWeb.parseURL(target_url).hostname;
} catch(ee) {
return false;
}
if (source_hostname == target_hostname) {
// same domain, path could be relative
target_url_relative = '/';
var v = target_url_no_protocol.split('/');
if (v.length>1) target_url_relative = '/' + v.slice(1, v.length).join('/')
}
//var html = cd.documentElement.innerHTML;
//var ind1 = html.indexOf(target_url_no_protocol);
var found = false;
var links = cd.documentElement.getElementsByTagName('a');
for(var i=0;i<links.length;i++) {
try {
var link = links[i].href;
link = link.replace(/^http(s?)\:\/\//, '');
if (link == target_url_no_protocol) {
found = true;
break;
}
else {
if (target_url_relative && link == target_url_relative) {
found = true;
break;
}
}
} catch(ee) {}
}
return found;
} catch(ee) {
_log("Error on hasURL: " + ee);
return false;
}
},
/*
getMetaRefresh: function(cd, url){
var metas = null;
var redURL = null;
var title = null;
try{redURL = cd.split('URL=')[1].split('>')[0].replace('"','').replace("'","")}catch(ee){};
CliqzHumanWeb.httpCache[url] = {'status': '301', 'time': CliqzHumanWeb.counter, 'location': redURL};
//Get first redirection.. for yahoo and stuff
if(url.indexOf('r.search.yahoo.com') > -1){
try{var _url = CliqzHumanWeb.linkCache[decodeURIComponent(url)]['s']}catch(ee){var _url = url}
CliqzHumanWeb.linkCache[decodeURIComponent(redURL.replace("'",""))] = {'s': ''+_url, 'time': CliqzHumanWeb.counter};
}
return redURL;
},
*/
eventDoorWayPage: function(cd){
var payload = {};
var url = cd.location.href;
var doorwayURL = cd.getElementsByTagName('a')[0].href;
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){};
var orignalDomain = CliqzHumanWeb.parseURL(url).hostname;
var dDomain = CliqzHumanWeb.parseURL(doorwayURL).hostname;
if(orignalDomain == dDomain) return;
payload = {"url":url, "durl":doorwayURL,"ctry": location};
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'doorwaypage', 'payload': payload});
},
getPageData: function(url, cd) {
var len_html = null;
var len_text = null;
var title = null;
var numlinks = null;
var inputs = null;
var inputs_nh = null;
var inputs_pwd = null;
var frames_same_host = null;
var iframes_same_host = null;
var frames = null;
var forms = null;
var pg_l = null;
var metas = null;
var tag_html = null;
var iall = true;
var all = null;
var canonical_url = null;
var url_host = null;
var frame_host = null;
try {
url_host = CliqzHumanWeb.parseURL(url).hostname;
} catch(ee) {
url_host = null;
}
try { len_html = cd.documentElement.innerHTML.length; } catch(ee) {}
try { len_text = cd.documentElement.textContent.length; } catch(ee) {}
try { title = cd.getElementsByTagName('title')[0].textContent; } catch(ee) {}
//title = unescape(encodeURIComponent(title));
try { numlinks = cd.getElementsByTagName('a').length; } catch(ee) {}
try {
inputs = cd.getElementsByTagName('input') || [];
inputs_nh = 0;
inputs_pwd = 0;
for(var i=0;i<inputs.length;i++) {
if (inputs[i]['type'] && inputs[i]['type']!='hidden') inputs_nh+=1;
if (inputs[i]['type'] && inputs[i]['type']=='password') inputs_pwd+=1;
}
} catch(ee) {}
try {
frames = cd.getElementsByTagName('frame') || [];
frames_same_host = 0;
for(var i=0;i<frames.length;i++) {
if (frames[i]['src']) {
var tsrc = frames[i]['src'];
if (frames[i]['src'].startsWith('//')) tsrc = 'http:' + frames[i]['src'];
try {
frame_host = CliqzHumanWeb.parseURL(tsrc).hostname;
if (frame_host===url_host || frame_host=='browser') frames_same_host++;
} catch(ee) {
frames_same_host++;
}
}
}
}
catch(ee) {}
try {
frames = cd.getElementsByTagName('iframe') || [];
iframes_same_host = 0;
for(var i=0;i<frames.length;i++) {
if (frames[i]['src']) {
var tsrc = frames[i]['src'];
if (frames[i]['src'].startsWith('//')) tsrc = 'http:' + frames[i]['src'];
try {
frame_host = CliqzHumanWeb.parseURL(tsrc).hostname;
if (frame_host===url_host || frame_host=='browser') iframes_same_host++;
} catch(ee) {
iframes_same_host++;
}
}
}
}
catch(ee) {}
try { forms = cd.getElementsByTagName('form'); } catch(ee) {}
//Detect doorway pages
// TBF : Need to make detecting of doorway page more strong. Currently lot of noise getting through.
/*
if(numlinks == 1 && cd.location){
CliqzHumanWeb.eventDoorWayPage(cd);
}
*/
var metas = cd.getElementsByTagName('meta');
// extract the language of th
try {
for (let i=0;i<metas.length;i++) {if (metas[i].getAttribute("http-equiv") == "content-language" || metas[i].getAttribute("name") == "language") {
pg_l = metas[i].getAttribute("content");
}};
if (pg_l == null) {
tag_html = cd.getElementsByTagName('html');
pg_l = tag_html[0].getAttribute("lang");
};
}catch(ee){}
// extract if indexable, no noindex on robots meta tag
try {
for (let i=0;i<metas.length;i++) {
var cnt = metas[i].getAttribute('content').toLowerCase();
if (cnt!=null && cnt.indexOf('noindex') > -1) {
iall = false;
}
}
}catch(ee){}
// extract the canonical url if available
var link_tag = cd.getElementsByTagName('link');
for (var j=0;j<link_tag.length;j++) {
if (link_tag[j].getAttribute("rel") == "canonical") {
canonical_url = link_tag[j].href;
// This check is done because of misplaces titles on sites like 500px, youtube etc.
// Since could not find a proper fix, hence dropping canonical URL looks like a safe idea.
if(CliqzHumanWeb.can_url_match[canonical_url] && CliqzHumanWeb.can_url_match[canonical_url] != url) canonical_url = null;
}
}
if (canonical_url != null && canonical_url.length > 0) {
// check that is not relative
if (canonical_url[0] == '/') {
try {
var ourl = CliqzHumanWeb.parseURL(url);
// ignore if httpauth or if non standard port
canonical_url = ourl['protocol'] + '://' + ourl['hostname'] + canonical_url;
} catch(ee) {}
}
}
// extract the location of the user (country level)
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){}
var x = {'lh': len_html, 'lt': len_text, 't': title, 'nl': numlinks, 'ni': (inputs || []).length, 'ninh': inputs_nh, 'nip': inputs_pwd, 'nf': (forms || []).length, 'pagel' : pg_l , 'ctry' : location, 'iall': iall, 'canonical_url': canonical_url, 'nfsh': frames_same_host, 'nifsh': iframes_same_host};
return x;
},
getHTML(url) {
// lazy load fallback urls
const possibleUrls = [
url => decodeURI(decodeURI(url)),
url => decodeURIComponent(url),
];
function getDoc(url) {
return getHTML(url).then( docs => {
const doc = docs[0];
if (doc) {
return doc;
} else {
throw "not found";
}
}).catch( () => {
const possibleUrl = possibleUrls.shift();
if (possibleUrl) {
const url = possibleUrl();
return getDoc(url);
} else {
throw "not found";
}
});
}
return getDoc(url);
},
createDoc(html) {
const hiddenWindow = Services.appShell.hiddenDOMWindow;
const parser = new hiddenWindow.DOMParser();
return parser.parseFromString(html, "text/html");
},
getCD(url) {
return this.getHTML(url).then( html => {
const doc = this.createDoc(html);
// monkey patching the doc so it looks like regular document element
/*
doc.location = {
href: url
};
*/
return doc;
});
},
captureJSRefresh: function(aRequest, aURI){
if(aRequest && aRequest.referrer) {
var refU = aRequest.referrer.asciiSpec;
var newURL = aURI.spec;
if(refU.indexOf('t.co/') > -1) CliqzHumanWeb.httpCache[refU] = {'status': '301', 'time': CliqzHumanWeb.counter, 'location': aURI.spec};
//Get first redirection.. for yahoo and stuff
var refyahoo = /\.search.yahoo\..*RU=/;
if(refyahoo.test(refU)){
try{var _url = CliqzHumanWeb.linkCache[decodeURIComponent(refU)]['s']}catch(ee){var _url = refU}
CliqzHumanWeb.linkCache[newURL] = {'s': ''+_url, 'time': CliqzHumanWeb.counter};
}
}
},
listener: {
tmpURL: undefined,
QueryInterface: XPCOMUtils.generateQI(["nsIWebProgressListener", "nsISupportsWeakReference"]),
onLocationChange: function(aProgress, aRequest, aURI) {
// New location, means a page loaded on the top window, visible tab
// Return if it's a private tab.
if(aRequest && aRequest.isChannelPrivate !== undefined && aRequest.isChannelPrivate) {
return;
}
if(aProgress.isLoadingDocument){
CliqzHumanWeb.captureJSRefresh(aRequest, aURI);
}
if (aURI.spec == this.tmpURL) return;
this.tmpURL = aURI.spec;
// This code looks obselete now, will remove in 1+ release.
if (CliqzHumanWeb.ismRefresh){
try{
var tabID = CliqzHumanWeb.getTabID();
if(tabID){
var mrefreshUrl = CliqzHumanWeb.mRefresh[tabID];
var parentRef = CliqzHumanWeb.linkCache[mrefreshUrl]['s'];
CliqzHumanWeb.linkCache[decodeURIComponent(aURI.spec)] = {'s': ''+mrefreshUrl, 'time': CliqzHumanWeb.counter};
CliqzHumanWeb.state['v'][mrefreshUrl]['qr'] = CliqzHumanWeb.state['v'][parentRef]['qr'];
if(CliqzHumanWeb.state['v'][mrefreshUrl]['qr']){
//Change type to ad, else might create confusion.
CliqzHumanWeb.state['v'][mrefreshUrl]['qr']['t'] = 'gad';
}
CliqzHumanWeb.ismRefresh = false;
delete CliqzHumanWeb.mRefresh[tabID];
}
}
catch(ee){};
}
// here we check if user ignored our results and went to google and landed on the same url
var requery = /\.google\..*?[#?&;]q=[^$&]+/; // regex for google query
var yrequery = /.search.yahoo\..*?[#?&;]p=[^$&]+/; // regex for yahoo query
var brequery = /\.bing\..*?[#?&;]q=[^$&]+/; // regex for yahoo query
var reref = /\.google\..*?\/(?:url|aclk)\?/; // regex for google refurl
var rerefurl = /url=(.+?)&/; // regex for the url in google refurl
// var gadurl = /\.google..*?\/(aclk)\?/;
// suggested by AMO - requires further testing in case of rapid tab switching (orange, blue)
// original:
// var currwin = aProgress.topWindow || CliqzUtils.getWindow().gBrowser.selectedBrowser.contentDocument;
// currwin is not used anymore, since the document is fetched using content script.
// var currwin = aProgress.DOMWindow.top;
// if(!currwin) return; //internal FF page
// This code looks obselete now, will remove in 1+ release.
if(gadurl.test(aURI.spec)){
var tabID = CliqzHumanWeb.getTabID();
if(tabID){
CliqzHumanWeb.ismRefresh = true;//{'status': '301', 'time': CliqzHumanWeb.counter, 'location': decodeURIComponent(CliqzHumanWeb.parseUri(aURI.spec)['queryKey']['adurl'])};
CliqzHumanWeb.mRefresh[tabID] = decodeURIComponent(aURI.spec);
}
}
// var currwin = CliqzUtils.getWindow();
// var _currURL = '' + currwin.gBrowser.selectedBrowser.contentDocument.location;
/*
//This needs to go away. Should get the content from contentDocument, but it is coming as null right now.
if(_currURL.indexOf('t.co/') > -1){
CliqzUtils.httpGet(_currURL,
function(res){
if(res && res.response){
try {
var _metaCD = res.response;
var redURL = CliqzHumanWeb.getMetaRefresh(_metaCD,_currURL );
} catch(e){}
}
}, null, 2000);
}
else if(_currURL.indexOf('r.search.yahoo.com') > -1){
CliqzUtils.httpGet(_currURL,
function(res){
if(res && res.response){
try {
var _metaCD = res.response;
var redURL = CliqzHumanWeb.getMetaRefresh(_metaCD,_currURL );
} catch(e){}
}
}, null, 2000);
}
else if(gadurl.test(_currURL)){
var tabID = CliqzHumanWeb.getTabID();
if(tabID){
CliqzHumanWeb.ismRefresh = true;//{'status': '301', 'time': CliqzHumanWeb.counter, 'location': decodeURIComponent(CliqzHumanWeb.parseUri(_currURL)['queryKey']['adurl'])};
CliqzHumanWeb.mRefresh[tabID] = decodeURIComponent(_currURL);
}
}
*/
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActiveAll = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(aURI.spec);
//Check if the URL is know to be bad: private, about:, odd ports, etc.
if (CliqzHumanWeb.isSuspiciousURL(activeURL)) return;
if (activeURL.indexOf('about:')!=0) {
if (CliqzHumanWeb.state['v'][activeURL] == null) {
//if ((requery.test(activeURL) || yrequery.test(activeURL) || brequery.test(activeURL) ) && !reref.test(activeURL)) {
var se = CliqzHumanWeb.checkSearchURL(activeURL);
if (se > -1) {
CliqzUtils.setTimeout(function(url) {
if (!CliqzHumanWeb) {
return;
}
CliqzHumanWeb.getCD(url).then( doc => {
CliqzHumanWeb.checkURL(doc, url, "normal");
CliqzHumanWeb.queryCache[url] = {
d: 0,
q: CliqzHumanWeb.searchCache[se]['q'],
t: CliqzHumanWeb.searchCache[se]['t']
};
let anonSe = CliqzHumanWeb.checkAnonSearchURL(url);
if(anonSe > -1){
try {
let hostName = CliqzHumanWeb.parseURL(url)['hostname'];
let qurl = "https://" + hostName + "/search?q=" + CliqzHumanWeb.searchCache[se]['q'];
let qObj = {};
qObj['qurl'] = qurl;
qObj['ts'] = Date.now();
qObj['tDiff'] = getRandomIntInclusive(1, 20);
CliqzHumanWeb.strictQueries.push(qObj);
CliqzHumanWeb.saveStrictQueries();
} catch(ee) {}
}
});
}, CliqzHumanWeb.WAIT_TIME, activeURL);
}
var status = null;
if (CliqzHumanWeb.httpCache[activeURL]!=null) {
status = CliqzHumanWeb.httpCache[activeURL]['status'];
}
var referral = null;
var qreferral = null;
if (CliqzHumanWeb.linkCache[activeURL] != null) {
//referral = CliqzHumanWeb.maskURL(CliqzHumanWeb.linkCache[activeURL]['s']);
referral = CliqzHumanWeb.linkCache[activeURL]['s'];
}
//Get redirect chain
var red = [];
red = CliqzHumanWeb.getRedirects(activeURL, red);
if(red.length == 0){
red = null;
}
//Set referral for the first redirect in the chain.
if (red && referral == null) {
var redURL = red[0];
var refURL = CliqzHumanWeb.linkCache[redURL];
if(refURL){
referral = refURL['s'];
}
//Update query cache with the redirected URL
if (CliqzHumanWeb.queryCache[redURL]) {
CliqzHumanWeb.queryCache[activeURL] = CliqzHumanWeb.queryCache[redURL];
}
}
CliqzHumanWeb.state['v'][activeURL] = {'url': activeURL, 'a': 0, 'x': null, 'tin': new Date().getTime(),
'e': {'cp': 0, 'mm': 0, 'kp': 0, 'sc': 0, 'md': 0}, 'st': status, 'c': [], 'ref': referral, 'red':red};
if (referral) {
// if there is a good referral, we must inherit the query if there is one
if (CliqzHumanWeb.state['v'][referral] && CliqzHumanWeb.state['v'][referral]['qr']) {
CliqzHumanWeb.state['v'][activeURL]['qr'] = {}
CliqzHumanWeb.state['v'][activeURL]['qr']['q'] = CliqzHumanWeb.state['v'][referral]['qr']['q'];
CliqzHumanWeb.state['v'][activeURL]['qr']['t'] = CliqzHumanWeb.state['v'][referral]['qr']['t'];
CliqzHumanWeb.state['v'][activeURL]['qr']['d'] = CliqzHumanWeb.state['v'][referral]['qr']['d']+1;
//If the depth is greater then two, we need to check if the ref. is of same domain.
//If not then drop the QR object, else keep it.
if(CliqzHumanWeb.state['v'][activeURL]['qr']['d'] > 2){
delete CliqzHumanWeb.state['v'][activeURL]['qr'];
}
else if(CliqzHumanWeb.state['v'][activeURL]['qr']['d'] == 2){
try {
if(CliqzHumanWeb.parseURL(activeURL)['hostname'] != CliqzHumanWeb.parseURL(referral)['hostname']){
delete CliqzHumanWeb.state['v'][activeURL]['qr'];
}
} catch(ee) {
delete CliqzHumanWeb.state['v'][activeURL]['qr'];
}
}
}
}
// Change the type to gad if the it's coming from an ADV. else can give incorrect clicks on SERP results.
if(red && red.length > 0){
for(var i=0;i<red.length;i++){
if(gadurl.test(red[i]) && CliqzHumanWeb.state['v'][activeURL].hasOwnProperty('qr')){
CliqzHumanWeb.state['v'][activeURL]['qr']['t'] = 'gad';
}
}
}
CliqzUtils.setTimeout(function(currURL) {
// Extract info about the page, title, length of the page, number of links, hash signature,
// 404, soft-404, you name it
//
// we cannot get it directly via
// var cd = currWin.gBrowser.selectedBrowser.contentDocument;
// because during the time of the timeout there can be win or tab switching
//
//var activeURL = CliqzHumanWeb.currentURL();
//if (activeURL != currURL) {}
CliqzHumanWeb.getCD(currURL).then(function (cd) {
var se = CliqzHumanWeb.checkSearchURL(currURL);
if (se == -1){
try {
CliqzHumanWeb.checkURL(cd, currURL,"normal");
} catch (e) {
}
//Check active usage...
CliqzHumanWeb.activeUsage += 1;
}
var x = CliqzHumanWeb.getPageData(currURL, cd);
if (x['canonical_url']) {
CliqzHumanWeb.can_urls[currURL] = x['canonical_url'];
CliqzHumanWeb.can_url_match[x['canonical_url']] = currURL;
}
if (CliqzHumanWeb.state['v'][currURL] != null) {
CliqzHumanWeb.state['v'][currURL]['x'] = x;
}
if (CliqzHumanWeb.queryCache[currURL]) {
CliqzHumanWeb.state['v'][currURL]['qr'] = CliqzHumanWeb.queryCache[currURL];
delete CliqzHumanWeb.queryCache[currURL];
}
if (CliqzHumanWeb.state['v'][currURL] != null) {
CliqzHumanWeb.addURLtoDB(currURL, CliqzHumanWeb.state['v'][currURL]['ref'], CliqzHumanWeb.state['v'][currURL]);
CliqzHumanWeb.queryCache[currURL];
}
}, function () {
if (CliqzHumanWeb.debug) {
_log("CANNOT GET THE CONTENT OF : " + currURL);
}
}).catch( ee => {
_log("Error fetching title and length of page: " + ee + " : " + currURL);
});
}, CliqzHumanWeb.WAIT_TIME, activeURL);
}
else {
// wops, it exists on the active page, probably it comes from a back button or back
// from tab navigation
CliqzHumanWeb.state['v'][activeURL]['tend'] = null;
CliqzEvents.pub('HW-activeURL:', {activeURL});
}
}
},
onStateChange: function(aWebProgress, aRequest, aFlag, aStatus) {
}
},
pacemaker: function() {
var activeURL = CliqzHumanWeb.currentURL();
if (activeURL && (activeURL).indexOf('about:')!=0) {
if ((CliqzHumanWeb.counter - CliqzHumanWeb.lastActive) < 5*CliqzHumanWeb.tmult) {
// if there has been an event on the last 5 seconds, if not do no count, the user must
// be doing something else,
//
try {
CliqzHumanWeb.state['v'][activeURL]['a'] += 1;
} catch(ee) {}
}
}
if ((activeURL==null) && ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % 10 == 0)) {
// this one is for when you do not have the page open, for instance, no firefox but console opened
CliqzHumanWeb.pushAllData();
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % 5 == 0) {
var openPages = CliqzHumanWeb.getAllOpenPages();
var tt = new Date().getTime();
for (var url in CliqzHumanWeb.state['v']) {
if (CliqzHumanWeb.state['v'].hasOwnProperty(url)) {
if (openPages.indexOf(url)==-1) {
// not opened
if (CliqzHumanWeb.state['v'][url]['tend']==null) {
CliqzHumanWeb.state['v'][url]['tend'] = tt;
}
if ((tt - CliqzHumanWeb.state['v'][url]['tend']) > CliqzHumanWeb.deadFiveMts*60*1000) {
// move to "dead pages" after 5 minutes
CliqzHumanWeb.state['m'].push(CliqzHumanWeb.state['v'][url]);
CliqzHumanWeb.addURLtoDB(url, CliqzHumanWeb.state['v'][url]['ref'], CliqzHumanWeb.state['v'][url]);
delete CliqzHumanWeb.state['v'][url];
delete CliqzHumanWeb.queryCache[url];
}
}
else {
// stil opened, do nothing.
if ((tt - CliqzHumanWeb.state['v'][url]['tin']) > CliqzHumanWeb.deadTwentyMts*60*1000) {
// unless it was opened more than 20 minutes ago, if so, let's move it to dead pages
CliqzHumanWeb.state['v'][url]['tend'] = null;
CliqzHumanWeb.state['v'][url]['too_long'] = true;
CliqzHumanWeb.state['m'].push(CliqzHumanWeb.state['v'][url]);
CliqzHumanWeb.addURLtoDB(url, CliqzHumanWeb.state['v'][url]['ref'], CliqzHumanWeb.state['v'][url]);
delete CliqzHumanWeb.state['v'][url];
delete CliqzHumanWeb.queryCache[url];
//_log("Deleted: moved to dead pages after 20 mts.");
//_log("Deleted: moved to dead pages after 20 mts: " + CliqzHumanWeb.state['m'].length);
}
}
}
}
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % 10 == 0) {
if (CliqzHumanWeb.debug) {
_log('Pacemaker: ' + CliqzHumanWeb.counter/CliqzHumanWeb.tmult + ' ' + activeURL + ' >> ' + CliqzHumanWeb.state.id);
}
// Ensuring the dbConn is not null.
var dbConn = CliqzHumanWeb.getDBConn();
CliqzHumanWeb.cleanHttpCache();
CliqzHumanWeb.cleanDocCache();
CliqzHumanWeb.cleanLinkCache();
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % (1*60) == 0) {
// every minute
CliqzHumanWeb.listOfUnchecked(1, CliqzHumanWeb.doubleFetchTimeInSec, null, CliqzHumanWeb.processUnchecks);
CliqzHumanWeb.auxGetQuery();
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % 10 == 0) {
var ll = CliqzHumanWeb.state['m'].length;
if (ll > 0) {
var v = CliqzHumanWeb.state['m'].slice(0, ll);
CliqzHumanWeb.state['m'] = CliqzHumanWeb.state['m'].slice(ll, CliqzHumanWeb.state['m'].length);
/*
for(var i=0;i<v.length;i++) {
CliqzHumanWeb.addURLtoDB(url, CliqzHumanWeb.state['v'][url]['ref'], CliqzHumanWeb.state['v'][url]);
}
*/
}
if(!CliqzHumanWeb.bloomFilter){
CliqzHumanWeb.loadBloomFilter();
}
}
//Load ts config
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % (60 * 20 * 1) == 0) {
if (CliqzHumanWeb.debug) {
_log('Load ts config');
}
CliqzHumanWeb.fetchAndStoreConfig();
CliqzHumanWeb.expireQuorumBloomFilter();
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % (60 * 60 * 1) == 0) {
if (CliqzHumanWeb.debug) {
_log('Check if alive');
}
CliqzHumanWeb.checkActiveUsage();
}
if ((CliqzHumanWeb.counter/CliqzHumanWeb.tmult) % (60 * 20 * 1) == 0) {
if (CliqzHumanWeb.debug) {
_log('Save actions stats');
}
CliqzHumanWeb.saveActionStats();
CliqzHumanWeb.sendActionStatsIfNeeded();
}
CliqzHumanWeb.counter += 1;
},
cleanUserTransitions: function(force) {
for(var query in CliqzHumanWeb.userTransitions['search']) {
if ((force) || ((CliqzHumanWeb.counter - CliqzHumanWeb.userTransitions['search'][query]['time']) > CliqzHumanWeb.userTransitionsSearchSession*CliqzHumanWeb.tmult)) {
// the query session is more than 5 minutes old or we are forcing the event,
// if the condition is met and there are more than two elements in data we
// must create the signal
//
if (CliqzHumanWeb.userTransitions['search'][query]['data'].length > 1) {
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){};
var doc = {'q': query, 'sources': CliqzHumanWeb.userTransitions['search'][query]['data'],'ctry': location};
if (CliqzHumanWeb.debug) {
_log(JSON.stringify(doc,undefined,2));
}
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'userTransition.search', 'payload': doc});
}
delete CliqzHumanWeb.userTransitions['search'][query];
}
}
},
pushAllData: function() {
// force send user Transitions sessions even if not elapsed because the browser is shutting down
CliqzHumanWeb.cleanUserTransitions(true);
var tt = new Date().getTime();
var res = [];
for (var url in CliqzHumanWeb.state['v']) {
if (CliqzHumanWeb.state['v'][url]) res.push(url);
}
for (var i=0; i<res.length; i++) {
// move all the pages to m set
var url = res[i];
if (CliqzHumanWeb.state['v'][url]) {
if (CliqzHumanWeb.state['v'][url]['tend']==null) {
CliqzHumanWeb.state['v'][url]['tend'] = tt;
}
CliqzHumanWeb.addURLtoDB(url, CliqzHumanWeb.state['v'][url]['ref'], CliqzHumanWeb.state['v'][url]);
CliqzHumanWeb.state['m'].push(CliqzHumanWeb.state['v'][url]);
delete CliqzHumanWeb.state['v'][url];
delete CliqzHumanWeb.queryCache[url];
}
}
// send them to telemetry if needed
var ll = CliqzHumanWeb.state['m'].length;
if (ll > 0) {
var v = CliqzHumanWeb.state['m'].slice(0, ll);
CliqzHumanWeb.state['m'] = CliqzHumanWeb.state['m'].slice(ll, CliqzHumanWeb.state['m'].length);
CliqzHumanWeb.pushTelemetry();
}
},
unload: function() {
//Check is active usage, was sent
try {var activeUsageTrk = CliqzUtils.getPref('config_activeUsage', null)} catch(ee){};
// Save action stats
CliqzHumanWeb.saveActionStats();
if(activeUsageTrk){
var tDiff = parseInt((new Date().getTime() - activeUsageTrk) / 1000);
if(tDiff && tDiff > 3600){
CliqzHumanWeb.checkActiveUsage();
}
else{
CliqzUtils.setPref('config_activeUsageCount', CliqzHumanWeb.activeUsage);
}
}
// send all the data
CliqzHumanWeb.pushTelemetry();
CliqzUtils.clearTimeout(CliqzHumanWeb.pacemakerId);
CliqzUtils.clearTimeout(CliqzHumanWeb.trkTimer);
},
unloadAtBrowser: function(){
try {
CliqzHumanWeb.activityDistributor.removeObserver(CliqzHumanWeb.httpObserver);
} catch(e){}
},
currentURL: function() {
var currwin = CliqzUtils.getWindow(), ret = null;
if (currwin && currwin.gBrowser) {
ret = currwin.gBrowser.selectedBrowser.currentURI.spec;
}
return CliqzHumanWeb.cleanCurrentUrl(ret);
},
cleanCurrentUrl: function(url){
try {
url = decodeURIComponent(url.trim());
} catch(ee) {}
if (url!=null || url!=undefined) return url;
else return null;
},
pacemakerId: null,
// load from the about:config settings
captureKeyPressPage: function(ev) {
if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['keypresspage']|0)) > 1 * CliqzHumanWeb.tmult) {
if (CliqzHumanWeb.debug) {
//_log('captureKeyPressPage');
}
CliqzHumanWeb.lastEv['keypresspage'] = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
CliqzHumanWeb.state['v'][activeURL]['e']['kp'] += 1;
}
}
},
captureMouseMovePage: function(ev) {
if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['mousemovepage']|0)) > 1 * CliqzHumanWeb.tmult) {
if (CliqzHumanWeb.debug) {
_log('captureMouseMovePage');
}
CliqzHumanWeb.lastEv['mousemovepage'] = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
CliqzHumanWeb.state['v'][activeURL]['e']['mm'] += 1;
}
}
},
getURLFromEvent: function(ev) {
try {
if (ev.target.href!=null || ev.target.href!=undefined) {
return decodeURIComponent(''+ev.target.href);
}
else {
if (ev.target.parentNode.href!=null || ev.target.parentNode.href!=undefined) {
return decodeURIComponent(''+ev.target.parentNode.href);
}
}
}
catch(ee) {
if (CliqzHumanWeb.debug) {
_log('Error in getURLFromEvent: ' + ee);
}
}
return null;
},
contextFromEvent: null,
setContextFromEvent: function(ev) {
try {
var tar = ev.target;
var found = false;
var count = 0;
var def_html = null;
while(!found) {
var html = tar.innerHTML;
if (html.indexOf('http://')!=-1 || html.indexOf('https://')!=-1) {
found = true;
def_html = html;
break;
}
tar = tar.parentNode;
count+=1;
if (count > 4) break;
}
if (found && def_html) {
CliqzHumanWeb.contextFromEvent = {'html': def_html, 'ts': (new Date()).getTime()};
}
}
catch(ee) {
CliqzHumanWeb.contextFromEvent = null;
}
},
captureMouseClickPage: function(ev, contextHTML) {
// if the target is a link of type hash it does not work, it will create a new page without referral
//
var targetURL = CliqzHumanWeb.getURLFromEvent(ev);
if (contextHTML) {
CliqzHumanWeb.contextFromEvent = {
html: contextHTML,
ts: Date.now()
};
} else {
CliqzHumanWeb.contextFromEvent = null;
}
if (targetURL!=null) {
var embURL = CliqzHumanWeb.getEmbeddedURL(targetURL);
if (embURL!=null) targetURL = embURL;
var activeURL = CliqzHumanWeb.currentURL();
if (CliqzHumanWeb.debug) {
_log('captureMouseClickPage>> ' + CliqzHumanWeb.counter + ' ' + targetURL + ' : ' + " active: " + activeURL + " " + (CliqzHumanWeb.state['v'][activeURL]!=null) + " " + ev.target + ' :: ' + ev.target.value + ' >>' + JSON.stringify(CliqzHumanWeb.lastEv));
}
//var activeURL = CliqzHumanWeb.currentURL();
if (CliqzHumanWeb.state['v'][activeURL]!=null) {
CliqzHumanWeb.linkCache[targetURL] = {'s': ''+activeURL, 'time': CliqzHumanWeb.counter};
//Fix same link in 'l'
//Only add if gur. that they are public and the link exists in the double fetch page(Public).it's available on the public page.Such
//check is not done, therefore we do not push the links clicked on that page. - potential record linkage.
//We need to check for redirections and use the final link for 'l' this is why the logic is here. This will
//for sure miss the first time it's see, cause we don't know on mouse click where it redirects.
var linkURL = targetURL;
if(CliqzHumanWeb.httpCache[targetURL]){
if(CliqzHumanWeb.httpCache[targetURL]['status'] == '301'){
linkURL = CliqzHumanWeb.httpCache[targetURL]['location'];
}
}
if(!CliqzHumanWeb.isSuspiciousURL(linkURL) && !CliqzHumanWeb.dropLongURL(linkURL)){
CliqzHumanWeb.getPageFromHashTable(linkURL, function(_res) {
if (_res && (_res['private'] == 0)) {
CliqzHumanWeb.state['v'][activeURL]['c'].push({'l': ''+ CliqzHumanWeb.maskURL(linkURL), 't': CliqzHumanWeb.counter});
}
else if(!_res){
CliqzHumanWeb.state['v'][activeURL]['c'].push({'l': ''+ CliqzHumanWeb.maskURL(linkURL), 't': CliqzHumanWeb.counter});
}
})
}
}
}
if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['mouseclickpage']|0)) > 1 * CliqzHumanWeb.tmult) {
if (CliqzHumanWeb.debug) {
_log('captureMouseClickPage');
}
CliqzHumanWeb.lastEv['mouseclickpage'] = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);;
if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
CliqzHumanWeb.state['v'][activeURL]['e']['md'] += 1;
}
}
},
captureScrollPage: function(ev) {
if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['scrollpage']|0)) > 1 * CliqzHumanWeb.tmult) {
if (CliqzHumanWeb.debug) {
_log('captureScrollPage ');
}
CliqzHumanWeb.lastEv['scrollpage'] = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
CliqzHumanWeb.state['v'][activeURL]['e']['sc'] += 1;
}
}
},
captureCopyPage: function(ev) {
if ((CliqzHumanWeb.counter - (CliqzHumanWeb.lastEv['copypage']|0)) > 1 * CliqzHumanWeb.tmult) {
if (CliqzHumanWeb.debug) {
_log('captureCopyPage');
}
CliqzHumanWeb.lastEv['copypage'] = CliqzHumanWeb.counter;
CliqzHumanWeb.lastActive = CliqzHumanWeb.counter;
var activeURL = CliqzHumanWeb.cleanCurrentUrl(ev.target.baseURI);
if (CliqzHumanWeb.state['v'][activeURL]!=null && CliqzHumanWeb.state['v'][activeURL]['a'] > 1*CliqzHumanWeb.tmult) {
CliqzHumanWeb.state['v'][activeURL]['e']['cp'] += 1;
}
}
},
counter: 0,
tmult: 4,
tpace: 250,
lastEv: {},
lastActive: null,
lastActiveAll: null,
getAllOpenPages: function() {
const urls = [];
try {
forEachWindow(win => {
const openTabs = queryActiveTabs(win);
openTabs.forEach(data => {
const url = data.url;
if (url && urls.indexOf(url) === -1 && url.startsWith('about:') === false) {
urls.push(decodeURIComponent(url));
}
});
});
} catch (ee) {
// do nothing, return empty set
}
return urls;
/*
var res = [];
try {
var enumerator = Services.wm.getEnumerator('navigator:browser');
while (enumerator.hasMoreElements()) {
var win = enumerator.getNext();
var gBrowser = win.gBrowser;
if (gBrowser.tabContainer) {
var numTabs = gBrowser.tabContainer.childNodes.length;
for (var i=0; i<numTabs; i++) {
var currentTab = gBrowser.tabContainer.childNodes[i];
var currentBrowser = gBrowser.getBrowserForTab(currentTab);
var currURL = currentBrowser.selectedBrowser.currentURI.spec;
if (currURL.indexOf('about:')!=0) {
res.push(decodeURIComponent(currURL));
}
}
}
}
return res;
}
catch(ee) {
return [];
}
*/
},
init: function(window) {
if(CliqzUtils.getPref("dnt", false)) return;
CliqzUtils.hw = this;
refineFuncMappings = {
"splitF":CliqzHumanWeb.refineSplitFunc,
"parseU":CliqzHumanWeb.refineParseURIFunc,
"maskU":CliqzHumanWeb.refineMaskUrl
};
_log("Init function called:")
var dbConn = CliqzHumanWeb.initDB();
if (CliqzHumanWeb.state == null) {
CliqzHumanWeb.state = {};
}
/*
else {
var util = window.QueryInterface(Components.interfaces.nsIInterfaceRequestor).getInterface(Components.interfaces.nsIDOMWindowUtils);
var win_id = util.outerWindowID;
if (CliqzHumanWeb.windowsMem[win_id] == null) {
CliqzHumanWeb.windowsMem[win_id] = window;
CliqzHumanWeb.windowsRef.push(window);
}
}
var util = window.QueryInterface(Components.interfaces.nsIInterfaceRequestor).getInterface(Components.interfaces.nsIDOMWindowUtils);
var win_id = util.outerWindowID;
if (CliqzHumanWeb.windowsMem[win_id] == null) {
CliqzHumanWeb.windowsMem[win_id] = window;
CliqzHumanWeb.windowsRef.push(window);
}
*/
if (CliqzHumanWeb.pacemakerId==null) {
CliqzHumanWeb.pacemakerId = CliqzUtils.setInterval(CliqzHumanWeb.pacemaker, CliqzHumanWeb.tpace, null);
}
CliqzHumanWeb.fetchAndStoreConfig();
if (CliqzHumanWeb.actionStats==null) CliqzHumanWeb.loadActionStats();
if (CliqzHumanWeb.actionStatsLastSent==null) CliqzHumanWeb.loadActionStatsLastSent();
// Load bloom filter
if(!CliqzHumanWeb.bloomFilter){
CliqzHumanWeb.loadBloomFilter();
}
// Load strict queries
CliqzHumanWeb.loadStrictQueries();
let rsNormal = new ResourceLoader(
["human-web","patterns"],
{
chromeURL: "chrome://cliqz/content/bower_components/patterns/index",
remoteURL : CliqzHumanWeb.patternsURL,
cron: 1 * 20 * 60 * 1000,
}
);
rsNormal.load().then( e => {
CliqzHumanWeb.loadContentExtraction(e, "normal")
});
rsNormal.onUpdate( e => CliqzHumanWeb.loadContentExtraction(e, "normal"));
let rsStrict = new ResourceLoader(
["human-web","patterns-anon"],
{
chromeURL: "chrome://cliqz/content/bower_components/anonpatterns/index",
remoteURL : CliqzHumanWeb.anonPatternsURL,
cron: 1 * 20 * 60 * 1000,
}
);
rsStrict.load().then( e => {
CliqzHumanWeb.loadContentExtraction(e, "strict")
});
rsStrict.onUpdate( e => CliqzHumanWeb.loadContentExtraction(e, "strict"));
// Load quorum config. Update every 20 minutes.
CliqzHumanWeb.safeQuorumProvider = new ResourceLoader(
["hw","quorum-config"],
{
remoteURL: CliqzHumanWeb.SAFE_QUORUM_PROVIDER,
cron: 1 * 20 * 60 * 1000,
updateInterval: 1 * 20 * 60 * 1000,
}
);
CliqzHumanWeb.safeQuorumProvider.load().then( e => {
CliqzHumanWeb.keyExpire = e.expiry * (24 * 60 * 60 * 1000); // Backend sends it in days;
CliqzHumanWeb.oc = e.oc;
CliqzHumanWeb.quorumThreshold = e.threshold;
});
CliqzHumanWeb.safeQuorumProvider.onUpdate(e => {
CliqzHumanWeb.keyExpire = e.expiry * (24 * 60 * 60 * 1000); // Backend sends it in days;
CliqzHumanWeb.oc = e.oc;
CliqzHumanWeb.quorumThreshold = e.threshold;
});
// Load quorum bloom filter
CliqzHumanWeb.loadQuorumBloomFilter();
},
initAtBrowser: function(){
if(CliqzUtils.getPref("dnt", false)) return;
CliqzHumanWeb.activityDistributor.addObserver(CliqzHumanWeb.httpObserver);
},
state: {'v': {}, 'm': [], '_id': Math.floor( Math.random() * 1000 ) },
hashCode: function(s) {
return s.split("").reduce(function(a,b){a=((a<<5)-a)+b.charCodeAt(0);return a&a},0);
},
msgSanitize: function(msg){
//Check and add time , else do not send the message
try {msg.ts = CliqzUtils.getPref('config_ts', null)} catch(ee){};
if(!msg.ts || msg.ts == ''){
return null;
}
// Adding anti-duplicate key, so to detect duplicate messages on the backend.
msg['anti-duplicates'] = Math.floor(Math.random() * 10000000);
if (msg.action == 'page') {
if(msg.payload.tend && msg.payload.tin){
var duration = msg.payload.tend - msg.payload.tin;
_log("Duration spent: " + msg.payload.tend + " : " + msg.payload.tin + " : " + duration );
}
else{
var duration = null;
_log("Duration spent: " + msg.payload.tend + " : " + msg.payload.tin + " : " + duration );
}
msg.payload['dur'] = duration;
delete msg.payload.tend;
delete msg.payload.tin;
// Check for fields which have urls like ref.
// Check if they are suspicious.
// Check if they are marked private.
if(msg.payload.ref){
if(CliqzHumanWeb.isSuspiciousURL(msg.payload['ref'])){
msg.payload['ref'] = null;
}
else{
msg.payload['ref'] = CliqzHumanWeb.maskURL(msg.payload['ref']);
}
// Check if ref. exists in bloom filter, then turn ref to null.
CliqzHumanWeb.getPageFromHashTable(msg.payload.ref, function(_res) {
if (_res) {
if(_res['private'] == 1 ){
msg.payload['ref'] = null;
}
}
})
}
// Remove continuations if not in the same domain for extra safety,
/*
if(msg.payload.c) {
var cleanCont = [];
try {
var mainDom = CliqzHumanWeb.parseURL(msg.payload.url).hostname;
msg.payload.c.forEach(function(e){
if (CliqzHumanWeb.parseURL(e.l).hostname == mainDom) cleanCont.push(e);
});
} catch(ee) {};
msg.payload.c = cleanCont;
}
// Check for title.
if(msg.payload.x.t){
if(CliqzHumanWeb.isSuspiciousTitle(msg.payload.x.t)){
_log("Suspicious Title: " + msg.payload.x.t);
return null;
}
}
else{
_log("Missing Title: " + msg.payload.x.t);
return null;
}
*/
// Remove C
if(msg.payload.c) {
msg.payload.c = null;
}
if (CliqzHumanWeb.dropLongURL(msg.payload.url)==true) {
// the url is not safe, replace by the canonical if exists and is safe
var canonical_url = msg.payload.x.canonical_url;
if (canonical_url != null && canonical_url != '' && CliqzHumanWeb.dropLongURL(canonical_url)==false) {
// the canonical exists and is ok
msg.payload.url = canonical_url;
}
else {
_log("Suspicious url with no/bad canonical: " + msg.payload.url);
return null;
}
}
else {
var canonical_url = msg.payload.x.canonical_url;
if (canonical_url != null && canonical_url != '' && CliqzHumanWeb.dropLongURL(canonical_url)==true) {
// the canonical is not safe, but the url is, remove only the canonical
msg.payload.x.canonical_url = null;
}
}
// validate that is not a shortener url, they are not useful anyway
var short_url = CliqzHumanWeb.isShortenerURL(msg.payload.url);
var short_canonical_url = false;
if (msg.payload.x.canonical_url != null && msg.payload.x.canonical_url != '') {
short_canonical_url = CliqzHumanWeb.isShortenerURL(msg.payload.x.canonical_url);
}
if (short_url || short_canonical_url) return null;
// check if suspiciousURL
if (CliqzHumanWeb.isSuspiciousURL(msg.payload.url)) return null;
if (msg.payload.x.canonical_url != null && msg.payload.x.canonical_url != '') {
if (CliqzHumanWeb.isSuspiciousURL(msg.payload.x.canonical_url)) return null;
}
//Mask the long ugly redirect URLs
if(msg.payload.red){
var cleanRed = [];
msg.payload.red.forEach(function(e){
cleanRed.push(CliqzHumanWeb.maskURL(e));
})
msg.payload.red = cleanRed;
}
// Check for canonical seen or not.
if(msg.payload['x']['canonical_url']) {
if(msg.payload['url'] == msg.payload['x']['canonical_url']){
_log("Canoncial is same: ");
// canonicalSeen = CliqzHumanWeb.canoincalUrlSeen(msg.payload['x']['canonical_url']);
if(msg.payload['csb'] && msg.payload['ft']) {
_log("Canoncial seen before: ");
delete msg.payload.csb;
delete msg.payload.ft;
}
}
// if the url is not replaces by canonical then also clear the csb key.
if(msg.payload['csb']) delete msg.payload.csb;
}
}
// FIXME: this cannot be here, telemetry is only for sending logic. The object needs to be
// handled beforehand!!!
// Canonical URLs and Referrals.
/*
if(CliqzHumanWeb.can_urls[msg.payload.url]){
msg.payload.url = CliqzHumanWeb.can_urls[msg.payload.url];
}
*/
//Check the depth. Just to be extra sure.
if(msg.payload.qr){
if(msg.payload.qr.d > 2){
delete msg.payload.qr;
}
}
// Check if qr.q is suspicious.
if(msg.payload.qr){
if (CliqzHumanWeb.isSuspiciousQuery(msg.payload.qr.q)) {
delete msg.payload.qr;
}
}
//Check for doorway action durl
if(msg.action=='doorwaypage') {
if((CliqzHumanWeb.isSuspiciousURL(msg.payload['durl'])) || (CliqzHumanWeb.isSuspiciousURL(msg.payload['url']))){
return null;
}
if((CliqzHumanWeb.dropLongURL(msg.payload['durl'])) || (CliqzHumanWeb.dropLongURL(msg.payload['url']))){
return null;
}
}
//Remove the msg if the query is too long,
if(msg.action=='query' || msg.action == 'anon-query') {
//Remove the msg if the query is too long,
if ((msg.payload.q == null) || (msg.payload.q == '')) {
return null;
}
else {
if (CliqzHumanWeb.isSuspiciousQuery(msg.payload.q)) {
return null;
}
}
}
return msg;
},
// ****************************
// telemetry, PREFER NOT TO SHARE WITH CliqzUtils for safety, blatant rip-off though
// ****************************
trk: [],
trkTimer: null,
notification: function(payload){
try {var location = CliqzUtils.getPref('config_location', null)} catch(ee){};
if(payload && typeof(payload) === 'object'){
payload['ctry'] = location;
CliqzHumanWeb.telemetry({'type': CliqzHumanWeb.msgType, 'action': 'telemetry', 'payload': payload});
}
else{
_log("Not a valid object, not sent to notification"