ermakovpetr

yjtDZx3mLJRsCWsACdN2yDNPW72tunRk8tXk#qLKim47aygZ6Qst3uabWf6yjXoG

ermakovpetr

- name: Setup GCC
  hosts: jupyterhubs_main
  tasks:

  - name: Create dirs for gcc build
    file:
      path: /tmp/tmp_for_gcc
      state: directory
    become: yes
    become_user: jupyterhub_user

ermakovpetr

----views between boosts
drop table lushpina.y_511_views_between_boosts;
create table lushpina.y_511_views_between_boosts as
select
dt1,
from_unixtime(unix_timestamp(date_created, 'yyyy-MM-dd hh:mm:ss.0'), 'EEEE') week_day_created,
y_511_tmp_lagged_boosts_w_all_dates.product_id,
seller_type,
floor((unix_timestamp(event_timestamp, 'yyyy-MM-dd hh:mm:ss.0')-unix_timestamp(boost, 'yyyy-MM-dd hh:mm:ss.0'))/60/60) diff_view_after_boost_hours,
floor((unix_timestamp(event_timestamp, 'yyyy-MM-dd hh:mm:ss.0')-unix_timestamp(date_created, 'yyyy-MM-dd hh:mm:ss.0'))/60/60/24) diff_view_after_create_days,

ermakovpetr

import telepot
import warnings
from IPython.core.magic import Magics, magics_class, line_magic, cell_magic

class TelegramSender(object):
    
    def __init__(self, token, user_id, prefix='', max_retries=3):
        self.user_id = user_id
        self.bot = telepot.Bot(token)
        self.prefix = prefix

ermakovpetr

Launching java with spark-submit command spark-submit --jars /home/z.lushpina/spark-cassandra-guava.jar  --driver-memory "2g" sparkr-shell /tmp/Rtmpkpwqf8/backend_port16145463a649 
Error in invokeJava(isStatic = TRUE, className, methodName, ...): org.apache.spark.SparkException: Yarn application has already ended! It might have been killed or unable to launch application master.
        at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.waitForApplication(YarnClientSchedulerBackend.scala:124)
        at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClientSchedulerBackend.scala:64)
        at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:144)
        at org.apache.spark.SparkContext.<init>(SparkContext.scala:541)
        at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:59)
        at org.apache.spark.api.r.RRDD$.createSparkContext(RRDD.scala:376)
        at org.apache.spark.api.r.RRDD.createSparkContext(RRDD.scala)

ermakovpetr

ermakovpetr

import time
import collections

import requests
from tqdm import tqdm


default_url = 'https://api.vk.com/method/{}' + \
              '&v=5.50&' + \
              'access_token=6ebe46f21ec2fca1ab326f34d88341c2979d4eba3850614ee73aaf0987d3da58fe71f4d6f314c7bb2661b'

ermakovpetr

latvian_surename.txt 999
--- latvian_surename.txt 0
--- holand_surename.txt 34
--- mongolian_surename.txt 1
--- georgian_surename.txt 6
--- arabic_surename.txt 3
--- spanish_surename.txt 4
--- ireland_surename.txt None
--- lituva_surename.txt 67
--- german_shurename.txt 18

ermakovpetr

      String userId = id.next();
      if (response.getStatusCode() == 302 || response.getStatusCode() == 404) {
        continue;
      }
      String responseBody = response.getResponseBody();
      Document docVKProfilePage = Jsoup.parse(responseBody);
      if (docVKProfilePage.select("div[class=profile_info_cont bl_item]").isEmpty()) {
        String serviceMsg = docVKProfilePage.select("div.service_msg").first().text();
        if (!"Страница удалена либо ещё не создана.".equals(serviceMsg)) {
          LOGGER.error("Error response from vk http, user: " + userId + " data: " + responseBody);

ermakovpetr

latvian_surename.txt

TOP 50 pos
vs= 5.19109
ns= 3.83031
ks= 3.77887
ica= 3.47301
js= 3.35853
ane= 2.98253
rs= 2.97397