Skip to content

Instantly share code, notes, and snippets.

View smalyshev's full-sized avatar

Stanislav Malyshev smalyshev

View GitHub Profile
w = Window.partitionBy(F.col('ip'), F.col('year'), F.col('month'), F.col('day'))
results = (
spark.read.table('wmf.webrequest')
.where(row_timestamp > ts_start)
.where(row_timestamp < ts_end)
.where(F.col('webrequest_source') == 'text')
.where(F.col('uri_host') == 'query.wikidata.org')
.where(F.col('cache_status') == 'miss')
.where(F.col('uri_path') != '/bigdata/ldf')
.where(F.col('http_status') == '200')
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>org.linkeddatafragments</groupId>
<artifactId>ldfserver</artifactId>
<version>0.2.3-wmf</version>
<packaging>war</packaging>
<name>Linked Data Fragments Server</name>
<url>http://linkeddatafragments.org</url>
<description>
Dump Journal
Running...
magic=e6b4c275
version=1
extent=714218864640(681132M), userExtent=714218863952(681132M), bytesAvailable=-46672592305776051(-44510452562M), nextOffset=46673306524640003
rootBlock{ rootBlock=0, challisField=20358252, version=3, nextOffset=47133885932561155, localTime=1546977056619 [Tuesday, January 8, 2019 7:50:56 PM UTC], firstCommitTime=1526550100463 [Thursday, May 17, 2018 9:41:40 AM UTC], lastCommitTime=1546977056616 [Tuesday, January 8, 2019 7:50:56 PM UTC], commitCounter=20358252, commitRecordAddr={off=NATIVE:-1675434151,len=422}, commitRecordIndexAddr={off=NATIVE:-1666221931,len=220}, blockSequence=1, quorumToken=-1, metaBitsAddr=7631697323500519, metaStartAddr=11989126, storeType=RW, uuid=1185498f-3aa5-4af8-90d1-1d0d3ba3985a, offsetBits=42, checksum=-632083319, createTime=1526550098831 [Thursday, May 17, 2018 9:41:38 AM UTC], closeTime=0}
rootBlock{ rootBlock=1, challisField=20358253, version=3, nextOffset=47133885932561156, localTime=1546977056702 [Tuesday, January 8, 2019 7:50
Dump Journal
Running...
magic=e6b4c275
version=1
extent=714218864640(681132M), userExtent=714218863952(681132M), bytesAvailable=-46672592305776051(-44510452562M), nextOffset=46673306524640003
rootBlock{ rootBlock=0, challisField=20357124, version=3, nextOffset=47133147198186243, localTime=1546976249604 [Tuesday, January 8, 2019 7:37:29 PM UTC], firstCommitTime=1526550100463 [Thursday, May 17, 2018 9:41:40 AM UTC], lastCommitTime=1546976249601 [Tuesday, January 8, 2019 7:37:29 PM UTC], commitCounter=20357124, commitRecordAddr={off=NATIVE:-991700643,len=422}, commitRecordIndexAddr={off=NATIVE:-639776809,len=220}, blockSequence=1, quorumToken=-1, metaBitsAddr=7631697323500330, metaStartAddr=11989126, storeType=RW, uuid=1185498f-3aa5-4af8-90d1-1d0d3ba3985a, offsetBits=42, checksum=-1344656082, createTime=1526550098831 [Thursday, May 17, 2018 9:41:38 AM UTC], closeTime=0}
rootBlock{ rootBlock=1, challisField=20357125, version=3, nextOffset=47133147198186244, localTime=1546976249706 [Tuesday, January 8, 2019 7:37:
rw-rw-r-- 1 blazegraph blazegraph 732G Jan 8 17:40 /srv/wdqs/wikidata.jnl
-rw-rw-r-- 1 blazegraph blazegraph 605G Jan 8 17:40 /srv/wdqs/wikidata.jnl
-rw-rw-r-- 1 blazegraph blazegraph 605G Jan 8 17:40 /srv/wdqs/wikidata.jnl
-rw-rw-r-- 1 blazegraph blazegraph 605G Jan 8 17:39 /srv/wdqs/wikidata.jnl
-rw-rw-r-- 1 blazegraph blazegraph 806G Jan 8 17:40 /srv/wdqs/wikidata.jnl
-rw-rw-r-- 1 blazegraph blazegraph 806G Jan 8 17:40 /srv/wdqs/wikidata.jnl
-rw-rw-r-- 1 blazegraph blazegraph 806G Jan 8 17:40 /srv/wdqs/wikidata.jnl
-rw-rw-r-- 1 blazegraph blazegraph 605G Jan 8 17:40 /srv/wdqs/wikidata.jnl
-rw-rw-r-- 1 blazegraph blazegraph 605G Jan 8 17:40 /srv/wdqs/wikidata.jnl
-rw-rw-r-- 1 blazegraph blazegraph 605G Jan 8 17:40 /srv/wdqs/wikidata.jnl
-------------------------
RWStore Allocator Summary
-------------------------
AllocatorSize AllocatorCount SlotsAllocated %SlotsAllocated SlotsRecycled SlotChurn SlotsInUse %SlotsInUse MeanAllocation SlotsReserved %SlotsUnused BytesReserved BytesAppData %SlotWaste %AppData %StoreFile %TotalWaste %FileWaste
64 63006 1382589955 14.05 1009463308 3.06 373126647 25.36 22 451627008 17.38 28904128512 23880105408 17.38 4.29 4.71 8.77 0.82
128 117294 876729080 8.91 60917784 1.04 815811296 55.44 93 840740352 2.97 107614765056 104423845888 2.97 18.77 17.54 5.57 0.52
192
Dump Journal
Running...
magic=e6b4c275
version=1
extent=864369901568(824327M), userExtent=864369900880(824327M), bytesAvailable=-52095946467309862(-49682566134M), nextOffset=52096810837210742
rootBlock{ rootBlock=0, challisField=41697920, version=3, nextOffset=52096810837210741, localTime=1546916729667 [Tuesday, January 8, 2019 3:05:29 AM UTC], firstCommitTime=1510337399915 [Friday, November 10, 2017 6:09:59 PM UTC], lastCommitTime=1546916729664 [Tuesday, January 8, 2019 3:05:29 AM UTC], commitCounter=41697920, commitRecordAddr={off=NATIVE:-47229760,len=422}, commitRecordIndexAddr={off=NATIVE:-2147462864,len=220}, blockSequence=1, quorumToken=-1, metaBitsAddr=8454741906433440, metaStartAddr=13189238, storeType=RW, uuid=84e0fc40-3d8f-47fb-8e01-c9899e2730f3, offsetBits=42, checksum=-227333422, createTime=1510337398696 [Friday, November 10, 2017 6:09:58 PM UTC], closeTime=0}
rootBlock{ rootBlock=1, challisField=41697921, version=3, nextOffset=52096810837210742, localTime=1546916729733 [Tuesday, January 8, 2019
Dump Journal
Running...
magic=e6b4c275
version=1
extent=864369901568(824327M), userExtent=864369900880(824327M), bytesAvailable=-52104437617654054(-49690663926M), nextOffset=52105301987554934
rootBlock{ rootBlock=0, challisField=41668718, version=3, nextOffset=52105301987554933, localTime=1546901828338 [Monday, January 7, 2019 10:57:08 PM UTC], firstCommitTime=1510337399915 [Friday, November 10, 2017 6:09:59 PM UTC], lastCommitTime=1546901828334 [Monday, January 7, 2019 10:57:08 PM UTC], commitCounter=41668718, commitRecordAddr={off=NATIVE:-2145194484,len=422}, commitRecordIndexAddr={off=NATIVE:-526146428,len=220}, blockSequence=1, quorumToken=-1, metaBitsAddr=8454741906433332, metaStartAddr=13189238, storeType=RW, uuid=84e0fc40-3d8f-47fb-8e01-c9899e2730f3, offsetBits=42, checksum=263532399, createTime=1510337398696 [Friday, November 10, 2017 6:09:58 PM UTC], closeTime=0}
rootBlock{ rootBlock=1, challisField=41668719, version=3, nextOffset=52105301987554934, localTime=1546901828460 [Monday, January 7, 2019 1
<http://www.wikidata.org/entity/Q24247028> <label> "Q24247028" .
<http://www.wikidata.org/entity/Q24247032> <label> "Q24247032" .
<http://www.wikidata.org/entity/Q24247033> <label> "Q24247033" .
<http://www.wikidata.org/entity/Q24247034> <label> "Q24247034" .
<http://www.wikidata.org/entity/Q24247037> <label> "Q24247037" .
<http://www.wikidata.org/entity/Q24247045> <label> "Q24247045" .
<http://www.wikidata.org/entity/Q24247047> <label> "Q24247047" .
<http://www.wikidata.org/entity/Q24247048> <label> "Q24247048" .
<http://www.wikidata.org/entity/Q24247052> <label> "Q24247052" .
<http://www.wikidata.org/entity/Q24247053> <label> "Q24247053" .
php-7.3.0alpha1.tar.gz
SHA256 hash: bf3d089d876ac817aecd6afca7ef801cda5c809d4954871e8e6be38bea656f2c
PGP signature:
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1
iF4EABEIAAYFAlsYI5EACgkQL3lWvF2gS13VYQD8CRnpb3777zrkHN9NAgCjv9xX
z4jBKIT7s4bxKpnlqh8A/jyhy9Ru6MoZkUwZWgE4FOTesW4Q50G1NCEuotLbTwOa
=87pc
-----END PGP SIGNATURE-----