sebbacon

require 'json'
require 'turbotlib'

Turbotlib.log("Starting scrape...")

state_file = open("#{Turbotlib.data_dir}/state.txt", "w")
(1...20).each do |n|
  data = {
    number: n,
    message: "Hello #{n}",

sebbacon

# Set up the index and type and mappings

curl -XPOST "http://localhost:9200/bork"
curl -XPOST "http://localhost:9200/bork/user/_mapping" -d '
{
  "user":{
    "properties":{
      "data_type":{
        "index":"not_analyzed",
        "type":"string"

sebbacon

require 'elasticsearch'
client = Elasticsearch::Client.new(:host => "elasticsearch-lb")
results = client.search(
  :index => 'openc_production',
  :type => 'company',
  :body => {
    "query"=> {
      "filtered"=> {
        "query"=> {
          "match_all"=> {}

sebbacon

ElasticsearchClient::INDEX = "openc_production"
 
query = {
  :type => "company",
  :body => {
    :filter => {
      :bool => {
        :should => [
          {
            :missing => {

sebbacon

# -*- coding: utf-8 -*-

require 'json'
require 'turbotlib'
require 'mechanize'

Turbotlib.log("Starting run...") # optional debug logging

url = 'http://www.cnv.gov.ar/Infofinan/BLOB_Zip.asp?cod_doc=257519&error_page=Error.asp'

sebbacon

set -e
curl -XDELETE http://localhost:9200/data/test
curl -XPUT http://localhost:9200/data/_mapping/test -d '
{
 "test" : {
   "properties" : {
    "License Status" : {"type": "string", "index" : "not_analyzed" },
    "License Type" : {"type": "string", "index" : "not_analyzed" },
    "License Expiration Date" : { "type" : "date", "format": "mm/dd/YYYY" }
   }

sebbacon

# test with
#
#    $ bundle exec hutch --require test.rb --verbose --mq-host rabbit1 --mq-api-port 55672
#
# Then publish a JSON message to the "testy" queue

class Test
  include Hutch::Consumer
  consume 'test.ack'
  queue_name 'testy'

sebbacon

Hutch::Config.set(:mq_host, 'rabbit1')
Hutch::Config.set(:mq_api_host, 'rabbit1')
Hutch::Config.set(:mq_api_port, 55672)
Hutch.connect
10000.times do
  Hutch.publish("angler.record", subject:'test')
end

sebbacon

for file in $(find . -name transformer.out); do
    echo " $file";
    jq -r 'select(.licence_holder.entity_type == "company" or (select(.licence_holder.entity_type == "unknown")|select(.licence_holder.entity_properties.name | test("\\b(LLC|LLLC|LLLP|LLC|INC|limited liability|incorporated|corporation|CORP|limited|LTD|PLLC|corp|llc|salon|sales|design|therapy|training|^Cnger|annual|institute|wine|beer|alcho|shop|[0-9#.]+|autos?|stores?|center|street|avenue|road|restaurant|east|west|north|south|central|club|pharmacy|carpet|electric|course|studio)\\b"; "i"))))|.permissions[].activity_name' $file >> /tmp/$(echo $file|cut -d "/" -f2).company.permissions;
done

sebbacon

#!/bin/bash

curl -XDELETE 'http://elasticsearch-lb:9200/business_licences/'
curl -XPOST 'http://elasticsearch-lb:9200/business_licences/'  -d '
{
  "settings":{
     "index":{
        "analysis":{
           "analyzer":{
              "analyzer_keyword":{