OriHoch

Vagrant.configure("2") do |config|
  config.vm.box = "ubuntu/trusty"
  config.vm.box_url = "https://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box"
  config.vm.provider "virtualbox" do |v|
    v.memory = 256
  end

  provision_script = <<-SCR
wget -O celerybeat https://raw.githubusercontent.com/celery/celery/master/extra/generic-init.d/celerybeat
cp celerybeat /etc/init.d/

OriHoch

import tabulator
from functools import partial

def _test_from_stream(stream, expected_content):
    try:
        stream.open()
    except tabulator.stream.exceptions.FormatError as e:
        if str(e) == "Format has been detected as HTML (not supported)":
            pass
        else:

OriHoch

#!/usr/bin/env python
from pymongo import MongoClient
import os
import requests

client = MongoClient(os.environ.get("MONGO_HOST", "localhost"), int(os.environ.get("MONGO_PORT", "27017")))
db = client[os.environ["MONGO_DB"]]
photoUnits = db['photoUnits']

data = requests.get("https://raw.githubusercontent.com/Beit-Hatfutsot/dbs-bagnowka-scrape/master/bagnowka_all.json").json()

OriHoch

search-result-exemption .company-stamp {
    line-height: 1.1em;
    width: 114px;
    height: 55px;
    position: absolute;
    top: 22px;
    right: 241px;
    background: url(assets/img/stamp-company.svg);
    transform: rotate(0deg);
    text-align: center;

OriHoch

from datapackage import Package
from tabulator import Stream
package = Package('https://minio.oknesset.org/committees/datapackage.json')
print(package.resource_names)
protocols_parsed = package.get_resource('committee_meeting_protocols_parsed')
for protocol_num, protocol in enumerate(protocols_parsed.iter(keyed=True)):
  print(protocol)
  with Stream("https://minio.oknesset.org/committees/" + protocol["parts_object_name"], headers=1) as stream:
    for part_num, part in enumerate(stream.iter(keyed=True)):
      print(part)

OriHoch

{
	"translatorID": "dcf19e16-0b1e-11e8-bed0-e4a4719186ba",
	"translatorType": 1,
	"label": "Migdar",
	"creator": "Ori Hoch",
	"target": "migdar",
	"minVersion": "3.0",
	"maxVersion": "",
	"priority": 100,
	"inRepository": false,

OriHoch

build_positions:
  pipeline:
  - run: load_resource
    parameters:
      url: data/datapackage.json
      resource: input_resource
  - run: split_resource
  - run: dump.to_path
    parameters:
      out-path: data/splitted_resource

OriHoch

#!/usr/bin/env bash

wget https://pypi.python.org/packages/6a/34/8176b841926a2add20524a9f74c307ac5fe6e33e9f4af12a58e6f7223982/mollyZ3950-2.04-molly1.tar.gz#md5=a0e5d7bb395ae31026afc7f974711630
sudo pip2 install ./mollyZ3950-2.04-molly1.tar.gz
sudo pip2 install pymarc

OriHoch

pip3 install -U datapackage-pipelines

OriHoch