Last active
January 12, 2022 12:56
-
-
Save rafaltrojanowski/1e678dcf39a1591c3b407cc091cd687b to your computer and use it in GitHub Desktop.
indexing issue
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Bastion: | |
| a1 = Domain.Repo.get_by(Domain.Answer, id: 555) | |
| query = %{"query" => %{"terms" => %{"_id" => [a1.id]}}} | |
| query |> Domain.Elasticsearch.search("answers_dev5") | |
| {:ok, | |
| %{ | |
| "_shards" => %{ | |
| "failed" => 0, | |
| "skipped" => 0, | |
| "successful" => 1, | |
| "total" => 1 | |
| }, | |
| "hits" => %{ | |
| "hits" => [ | |
| %{ | |
| "_id" => "555", | |
| "_index" => "answers_dev5", | |
| "_score" => 1.0, | |
| "_source" => %{ | |
| "additional_answers" => [], | |
| "comment" => "Karton wurde vom Paketdienst sicher geworfen.", | |
| "comment_de_DE" => "Karton wurde vom Paketdienst sicher geworfen.", | |
| "has_comment" => true, | |
| "id" => 555, | |
| "inserted_at" => "2021-06-08T09:31:32Z", | |
| "is_a_test" => false, | |
| "label_ids" => [9500], | |
| "language" => "de_DE", | |
| "language_id" => 75, | |
| "properties" => [], | |
| "public_hash_id" => "TXpZeFlUZGlaVFV0T1RRME5TMDBNV1EwTFRnNVpHWXRNakE1WWpGaE1USXlZV1Zr", | |
| "recipient_full_name" => nil, | |
| "recipient_id" => nil, | |
| "recipient_identity" => nil, | |
| "recipient_identity_type" => nil, | |
| "recipient_upload_method" => nil, | |
| "score" => 9, | |
| "score_type" => "promoter", | |
| "sentiment" => "positive", | |
| "sentiment_per_label" => [], | |
| "stripped_comment" => "Karton wurd Paketdien sich geworf", | |
| "survey_id" => 12, | |
| "survey_recipient_id" => nil, | |
| "translated_comment" => nil, | |
| "translated_language_id" => nil, | |
| "updated_at" => "2021-11-26T10:28:13Z" | |
| }, | |
| "_type" => "_doc" | |
| } | |
| ], | |
| "max_score" => 1.0, | |
| "total" => %{"relation" => "eq", "value" => 1} | |
| }, | |
| "timed_out" => false, | |
| "took" => 1 | |
| }} |
Dev2:
a1 = Domain.Repo.get_by(Domain.Answer, id: 1)
a1.label_ids_with_keywords
%{"24608" => ["voucher"]}
%{"24608" => ["voucher"]}
iex(api@10.4.3.78)25> query = %{"query" => %{"terms" => %{"_id" => [a1.id]}}}
%{"query" => %{"terms" => %{"_id" => [1]}}}
iex(api@10.4.3.78)26> query |> Domain.Elasticsearch.search("answers_dev5")
{:ok,
%{
"_shards" => %{
"failed" => 0,
"skipped" => 0,
"successful" => 1,
"total" => 1
},
"hits" => %{
"hits" => [
%{
"_id" => "1",
"_index" => "answers_dev5",
"_score" => 1.0,
"_source" => %{
"additional_answers" => [],
"comment" => "Hi Team, having problems with the voucher code…can you get back to me",
"has_comment" => true,
"id" => 1,
"inserted_at" => "2021-05-31T11:43:59Z",
"is_a_test" => nil,
"label_ids" => 'H',
"language" => nil,
"language_id" => nil,
"properties" => [
%{"id" => 4, "name" => "country", "value" => "uk"},
%{"id" => 7, "name" => "color", "value" => "white"},
%{"id" => 12, "name" => "value", "value" => "1350"},
%{"id" => 14, "name" => "currency", "value" => "GBP"}
],
"public_hash_id" => "TVdNeVlqZGpPR1F0Tm1Jd1lpMDBZMkkyTFRneVpETXROVEpsTldOaE5tWXpZalkw",
"recipient_full_name" => "Maxine Conley",
"recipient_id" => 1,
"recipient_identity" => "a.mi.fringilla@dolorFusce.org",
"recipient_identity_type" => "email",
"recipient_upload_method" => nil,
"score" => 8,
"score_type" => "passive",
"sentiment" => nil,
"sentiment_per_label" => [
%{"id" => 72, "name" => "Voucher", "sentiment" => "negative"}
],
"stripped_comment" => "Hi Team have problem voucher code can you get back me",
"survey_id" => 1,
"survey_recipient_id" => nil,
"translated_comment" => nil,
"translated_language_id" => nil,
"updated_at" => "2021-05-31T11:43:59Z"
},
"_type" => "_doc"
}
],
"max_score" => 1.0,
"total" => %{"relation" => "eq", "value" => 1}
},
"timed_out" => false,
"took" => 1
}}
Staging:
a1 = Domain.Repo.get_by(Domain.Answer, id: 1716536)
still missing label_ids_with_keywords
IEx.configure(inspect: [limit: :infinity])
query = from a in Domain.Answer, order_by: [desc: :id], limit: 1, select: a.id
%{
"active" => false,
"finished" => false,
"last_indexed_answer_id" => 1181044,
"page" => 1165,
"page_size" => 1000,
"started_at" => ~N[2021-12-21 10:38:56.086901],
"worker_pid" => #PID<0.2555.2>
}
iex(92)> Map.keys(body["answers"]["mappings"]["properties"]["label_ids_with_keywords"]["properties"]) |> Enum.count
54
iex(93)> Map.keys(body["answers"]["mappings"]["properties"]["label_ids_with_keywords"]["properties"])
["id", "8", "36", "107", "44", "100", "78", "71", "142", "120", "98", "64", "3",
"163", "93", "204", "148", "121", "155", "149", "23", "99", "225", "127", "86",
"one", "92", "50", "106", "119", "79", "65", "141", "keywords", "72", "29",
"96", "189", "30", "113", "134", "152", "135", "22", "162", "128", "37", "51",
"16", "114", ...]
iex(94)>
{:ok, response} = Domain.Env.get(:elasticsearch_url) |> URI.merge("/#{index}/_mapping?pretty") |> URI.to_string() |> Elastix.HTTP.get()
body = response.body
Map.keys(body["answers_staging_v3"]["mappings"]["properties"]["label_ids_with_keywords"]["properties"])
["749543", "780486", "749557", "757096", "355957", "658335", "350333", "673182",
"347694", "806265", "336233", "354701", "720378", "348914", "351208", "749571",
"9387", "419589", "813082", "347743", "353725", "344987", "351098", "345009",
"720364", "419613", "619025", "350312", "419626", "802585", "626760", "757374",
"460324", "776051", "355823", "345043", "351428", "749576", "516816", "349576",
"720392", "355638", "302052", "473966", "344985", "282497", "348333", "671581",
"302368", "806267", "647310", "336205", "353516", "336240", "3203", "3210",
"47093", "678677", "619045", "336214", "296732", "666942", "720374", "348923",
"303360", "336635", "799764", "749569", "815838", "349606", "806270", "345001",
"749519", "336753", "815825", "317215", "31", "809019", "354720", "347369",
"719062", "261708", "47597", "282496", "503843", "352409", "282457", "720404",
"682267", "334222", "282481", "749488", "688287", "282482", "282474", "804625",
"282495", "282455", "619010", "336184", "348337", "355624", "345049", "282501",
"3206", "336213", "647302", "749538", "749568", "351068", "354692", "355809",
"3207", "347415", "282504", "344995", "795312", "334193", "668060", "282494",
"804646", "302417", "719203", "347469", "353684", "6286", "47602", "282463",
"789725", "767527", "350321", "473964", "719041", "749567", "355915", "682265",
"749573", "749525", "618996", "749487", "461653", "345008", "719202", "353668",
"350850", "673224", "647295", "347724", "33", "350291", "518074", "351175",
"464646", "34", "353662", "503856", "794992", "647281", "354702", "305611",
"349583", "334215", "282489", "813668", "348343", "419563", "350292", "348360",
"815817", "282491", "354671", "282441", "336198", "719090", "353669", "749581",
"347462", "353678", "354713", "351379", "26", "21", "815796", "749523",
"302969", "749536", "282473", "350836", "261738", "302403", "282502", "344980",
"302361", "347525", "417904", "355945", "503878", "355617", "719204", "47085",
"348928", "720405", "354629", "806262", "348341", "677427", "282471", "282477",
"345042", "282509", "354594", "347492", "419593", "355978", "347182", "302971",
"806269", "282508", "671579", "334236", "374805", "282459", "666938", "348361",
"619064", "345021", "47539", "720377", "296733", "282465", "354818", "671580",
"282483", "673175", "749491", "348349", "347028", "345006", "3205", "47087",
"626799", "503925", "282490", "749545", "6287", "6291", "804644", "334208",
"719055", "302354", "355964", "626773", "626787", "673148", "336219", "355790",
"749589", "419597", "619017", "347757", "353712", "806261", "350340", "619038",
"302375", "671578", "811781", "749558", "302967", "265187", "282479", "804576",
"282462", "811822", "345010", "348889", "282487", "586583", "348903", "302410",
"815845", "414675", "749520", "282461", "282472", "282456", "647323", "351145",
"344992", "339543", "345029", "673161", "782327", "473963", "282451", "6289",
"351131", "348936", "719109", "719070", "353683", "811759", "317216", "35",
"749485", "348929", "355633", "348909", "3208", "347448", "302384", "647330",
"77196", "348315", "29", "626752", "348888", "350899", "749574", "749559",
"348904", "808304", "282484", "345973", "813672", "348874", "30", "774750",
"348632", "749544", "336189", "354706", "77191", "804605", "334229", "347497",
"647337", "460323", "302970", "811780", "619052", "334172", "351449", "339510",
"808256", "302656", "47537", "347715", "808298", "348319", "348887", "673210",
"626745", "22", "270139", "354734", "336226", "345022", "24", "749554",
"334182", "350296", "355950", "347355", "282486", "626731", "355603", "720365",
"806266", "336191", "799837", "47086", "336180", "336247", "626780", "348324",
"808235", "780487", "282470", "353502", "749512", "77190", "350354", "353693",
"385497", "720337", "37", "26958", "282507", "419592", "282505", "345028",
"668059", "345011", "815818", "345035", "347306", "282475", "673189", "334196",
"348382", "345896", "353711", "815810", "808284", "804590", "350866", "804597",
"668065", "808257", "719205", "282446", "334179", "19", "503927", "720406",
"419620", "647349", "349639", "719111", "355774"]
curl -X PUT "localhost:9200/answers_v1?pretty"
{
"acknowledged" : true,
"shards_acknowledged" : true,
"index" : "answers_v1"
}
curl -X POST "localhost:9200/_reindex?pretty" -H 'Content-Type: application/json' -d'
{
"source": {
"index": "answers"
},
"dest": {
"index": "answers_v1"
}
}
'
Question - 1) how to add new mapping here for v1??
2) It can take ages on prod, no?
curl -X POST "localhost:9200/_reindex?pretty" -H 'Content-Type: application/json' -d'
{
"source": {
"index": "answers"
},
"dest": {
"index": "answers_v1"
}
}
'
{
"took" : 13064,
"timed_out" : false,
"total" : 32563,
"updated" : 0,
"created" : 32563,
"deleted" : 0,
"batches" : 33,
"version_conflicts" : 0,
"noops" : 0,
"retries" : {
"bulk" : 0,
"search" : 0
},
"throttled_millis" : 0,
"requests_per_second" : -1.0,
"throttled_until_millis" : 0,
"failures" : [ ]
}
It looks like flattened was cached even after fetching the last changes. (probably before step 2 the mapping change need to be deployed or need to be done manually using mappings API like this (https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-put-mapping.html)
So I was able to reindex to new index.
Now how to delete old index and make an alias?
curl -X DELETE "localhost:9200/answers?pretty"
{
"acknowledged" : true
}
curl -X POST "localhost:9200/_aliases?pretty" -H 'Content-Type: application/json' -d'
{
"actions" : [
{ "add" : { "index" : "answers_v1", "alias" : "answers" } }
]
}
'
{
"acknowledged" : true
}
Status - no responses on UI. // edit ok after changing the scope
It works though: query |> Domain.Elasticsearch.search("answers")
iex(182)> query |> Domain.Elasticsearch.search("answers")
{:ok,
%{
"_shards" => %{
"failed" => 0,
"skipped" => 0,
"successful" => 1,
"total" => 1
},
"hits" => %{
"hits" => [
%{
"_id" => "811",
"_index" => "answers_v1",
"_score" => 1.0,
"_source" => %{
"additional_answers" => [],
"comment" => "Aktuell warte ich auf drei Hula Hoop Reifen. Im Februar habe ich zuerst einen bestellt mit Wartezeit - das war okay und wurde auf der Seite auch so dargestellt. Anfang März habe ich entdeckt, dass es ein zweier Set Hula Hoop Reifen gibt, bei welchen das Lieferdatum als GARANTIERT für den 9.4. ausgeschrieben war. (Screenshot vorhanden) Ich dachte mir schon, dass es sehr komisch ist, dass das 2er Set vor dem anderen kommen soll, aber gut. Abnehmer habe ich in meiner Familie gefunden. Zwischenzeitlich habe ich zig Emails bekommen, dass die Produkte nicht lieferbar sind. Beim 2er Set stand auf der Homepage vor ein paar Wochen, dass der garantierte Liefertermin am 1.4. ist, in der Email steht seit einiger Zeit, dass mit der nächsten Warenlieferung alle noch offenen Bestellungen beliefert werden. Ich fühle mich bei jeder einzelnen Email absolut VERARSCHT. Hier herrscht absolut keine ehrliche Kommunikation und so etwas werde ich nicht mehr unterstützen.",
"comment_de_DE" => "Aktuell warte ich auf drei Hula Hoop Reifen. Im Februar habe ich zuerst einen bestellt mit Wartezeit - das war okay und wurde auf der Seite auch so dargestellt. Anfang März habe ich entdeckt, dass es ein zweier Set Hula Hoop Reifen gibt, bei welchen das Lieferdatum als GARANTIERT für den 9.4. ausgeschrieben war. (Screenshot vorhanden) Ich dachte mir schon, dass es sehr komisch ist, dass das 2er Set vor dem anderen kommen soll, aber gut. Abnehmer habe ich in meiner Familie gefunden. Zwischenzeitlich habe ich zig Emails bekommen, dass die Produkte nicht lieferbar sind. Beim 2er Set stand auf der Homepage vor ein paar Wochen, dass der garantierte Liefertermin am 1.4. ist, in der Email steht seit einiger Zeit, dass mit der nächsten Warenlieferung alle noch offenen Bestellungen beliefert werden. Ich fühle mich bei jeder einzelnen Email absolut VERARSCHT. Hier herrscht absolut keine ehrliche Kommunikation und so etwas werde ich nicht mehr unterstützen.",
"has_comment" => true,
"id" => 811,
"inserted_at" => "2021-07-01T11:58:00Z",
"label_ids" => [92, 120, 162],
"label_ids_with_keywords" => %{
"120" => ["produkt"],
"162" => ["homepage", "seite"],
"92" => ["wartezeit"]
},
"language" => "de_DE",
"language_id" => 75,
"properties" => [],
"public_hash_id" => "TVduMkJhS21aU0hFWFc2OVpPbG04c0owbjd4SHdGb0ZkblJxaSsrMlFuSQ",
"recipient_full_name" => nil,
"recipient_id" => nil,
"recipient_identity" => nil,
"recipient_identity_type" => nil,
"recipient_upload_method" => nil,
"score" => 1,
"score_type" => "detractor",
"sentiment" => "positive",
"sentiment_per_label" => [
%{"id" => 120, "name" => "Sortiment", "sentiment" => "negative"},
%{"id" => 162, "name" => "Webseite", "sentiment" => "positive"},
%{"id" => 92, "name" => "Versand", "sentiment" => "positive"}
],
"stripped_comment" => "Aktuell wart drei Hula Hoop Reif Im Februar zuerst bestellt Wartezeit okay wurd Seit dargestellt Anfang Marz entdeckt dass zwei Set Hula Hoop Reif gibt Lieferdatum GARANTIERT 9.4 ausgeschrieb Screenshot vorhand Ich dacht schon dass komisch dass 2er Set komm gut Abnehm Famili gefund Zwischenzeitlich zig Email bekomm dass Produkt lieferbar Beim 2er Set stand Homepag paar Woch dass garantiert Liefertermin 1.4 Email steht seit Zeit dass nach Warenlieferung off Bestellung beliefert Ich fuhl einzeln Email absolut VERARSCHT Hier herrscht absolut ehrlich Kommunikation mehr unterstutz",
"survey_id" => 5,
"survey_recipient_id" => nil,
"translated_comment" => nil,
"translated_language_id" => nil,
"updated_at" => "2021-08-05T08:39:33Z"
},
"_type" => "_doc"
}
],
"max_score" => 1.0,
"total" => %{"relation" => "eq", "value" => 1}
},
"timed_out" => false,
"took" => 1
}}
But how about aliasing an ogranization special indexes???
custom indexes are in the following format:
answers_org_#{org_id}
Dev 2 steps:
https://github.com/zenloopGmbH/documentation/blob/master/docs/how_to/elasticsearch.md
template = {
"settings": {
"number_of_replicas": 1,
"number_of_shards": 1,
"refresh_interval": "1s",
"analysis": {
"analyzer": {
"lowercase_analyzer": {
"tokenizer": "lowercase"
},
"email_analyzer": {
"tokenizer": "uax_url_email"
}
}
}
},
"mappings": {
"properties": {
"id": {
"type": "integer"
},
"score": {
"type": "byte",
"null_value": 0
},
"stripped_comment": {
"type": "text",
"analyzer": "standard"
},
"comment": {
"type": "text",
"analyzer": "standard",
"search_analyzer": "standard",
"term_vector": "with_positions_offsets"
},
"comment_ar_AE": {
"analyzer": "arabic",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_bg_BG": {
"analyzer": "bulgarian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_bn_BD": {
"analyzer": "bengali",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ca_ES": {
"analyzer": "catalan",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_cs_CZ": {
"analyzer": "czech",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_da_DK": {
"analyzer": "danish",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_de_DE": {
"analyzer": "german",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_el_GR": {
"analyzer": "greek",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_en_US": {
"analyzer": "english",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_es_AR": {
"analyzer": "spanish",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_es_ES": {
"analyzer": "spanish",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_eu_ES": {
"analyzer": "basque",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_fr_FR": {
"analyzer": "french",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ga_IE": {
"analyzer": "irish",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_gl_ES": {
"analyzer": "galician",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_hi_IN": {
"analyzer": "hindi",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_hr_HR": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_hu_HU": {
"analyzer": "hungarian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_hy_AM": {
"analyzer": "armenian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_id_ID": {
"analyzer": "indonesian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_it_IT": {
"analyzer": "italian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ja_JP": {
"analyzer": "kuromoji",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ko_KR": {
"analyzer": "nori",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_lt_LT": {
"analyzer": "lithuanian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_lv_LV": {
"analyzer": "latvian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ms_MY": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_nl_NL": {
"analyzer": "dutch",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_pl_PL": {
"analyzer": "polish",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_pt_BR": {
"analyzer": "brazilian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_pt_PT": {
"analyzer": "portuguese",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ro_RO": {
"analyzer": "romanian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ru_RU": {
"analyzer": "russian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_sk_SK": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_sv_SE": {
"analyzer": "swedish",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_th_TH": {
"analyzer": "thai",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_tr_TR": {
"analyzer": "turkish",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_uk_UA": {
"analyzer": "ukrainian",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_vi_VN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_zh_Hans": {
"analyzer": "smartcn",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_af_ZA": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_sq_AL": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_am_ET": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_az_AZ": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_be_BY": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_bs_BA": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ceb_PH": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_co_FR": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_eo_EO": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_et_EE": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_fy_NL": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ka_GE": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_gu_IN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ht_HT": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ha_GH": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_haw_US": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_he_IL": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_hmn_HMN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_is_IS": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ig_NG": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_jv_ID": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_kn_IN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_kk_KZ": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_km_KH": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ku_IQ": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ky_KG": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_lo_LA": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_la_LA": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_lb_LU": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_mk_MK": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ma_MG": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_mg_MG": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ml_IN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_mt_MT": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_mi_MI": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_mr_IN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_mn_MN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ne_NP": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ny_ZW": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ps_AR": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_pa_IN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_sm_SM": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_gd_GB": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_sr_SR": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_sn_ZW": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_si_LK": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_sd_IN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_sl_SI": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_so_SO": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_su_ID": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_sw_KE": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_tl_PH": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_tg_TJ": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ta_IN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_te_IN": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_ur_PK": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_uz_AF": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_cy_GB": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_xh_ZA": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_yi": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_yo_NG": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"comment_zu_ZA": {
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"term_vector": "with_positions_offsets",
"type": "text"
},
"translated_comment": {
"type": "text",
"analyzer": "standard"
},
"score_type": {
"type": "keyword",
"null_value": "detractor"
},
"has_comment": {
"type": "boolean",
"null_value": false
},
"public_hash_id": {
"type": "keyword"
},
"properties": {
"type": "nested",
"properties": {
"id": {
"type": "integer"
},
"name": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"label_ids": {
"type": "keyword"
},
"label_ids_with_keywords": {
"type": "flattened"
},
"recipient_id": {
"type": "integer"
},
"recipient_full_name": {
"type": "text"
},
"recipient_identity": {
"type": "text",
"analyzer": "standard",
"search_analyzer": "lowercase_analyzer",
"fields": {
"email": {
"type": "text",
"analyzer": "email_analyzer",
"search_analyzer": "email_analyzer"
}
}
},
"recipient_identity_type": {
"type": "keyword"
},
"recipient_upload_method": {
"type": "keyword"
},
"sentiment": {
"type": "keyword"
},
"sentiment_per_label": {
"type": "nested",
"properties": {
"id": {
"type": "integer"
},
"name": {
"type": "keyword"
},
"sentiment": {
"type": "keyword"
}
}
},
"survey_id": {
"type": "integer"
},
"survey_recipient_id": {
"type": "integer"
},
"language": {
"type": "keyword"
},
"language_id": {
"type": "integer"
},
"translated_language_id": {
"type": "integer"
},
"inserted_at": {
"type": "date"
},
"updated_at": {
"type": "date"
},
"additional_answers": {
"type": "nested",
"properties": {
"id": {
"type": "text"
},
"additional_question_id": {
"type": "integer"
},
"additional_question_public_hash_id": {
"type": "keyword"
},
"answer": {
"type": "keyword"
},
"inserted_at": {
"type": "date"
},
"position": {
"type": "integer"
},
"question": {
"type": "text"
},
"score_system_id": {
"type": "integer"
},
"has_answer": {
"type": "boolean"
}
}
}
}
}
}
body = {
"index_patterns": [
"answers",
"answers*"
],
"template": {}
}
{:ok, response} = Domain.Env.get(:elasticsearch_url) |> URI.merge("/_index_template/answers_template/_mapping?pretty") |> URI.to_string() |> Elastix.HTTP.put()
Get all indexes:
{:ok, response} = Domain.Env.get(:elasticsearch_url) |> URI.merge("_cat/indices/answers*?v=true&s=index") |> URI.to_string() |> Elastix.HTTP.get()
iex(api@10.4.5.209)95> response.body
"health status index uuid pri rep docs.count docs.deleted store.size pri.store.size\ngreen open answers il41UIeMTKWWdTh1hwDC3g 1 1 5 0 21.6kb 10.8kb\ngreen open answers_dev1_v3 WP9W_71ERCCGN4UJgOV70Q 1 1 454111 187717 163.8mb 91.8mb\ngreen open answers_dev2_v3 kF81UqRMQxG5hqr8KOQMPw 1 1 485951 168523 190mb 93.2mb\ngreen open answers_dev3_v1 Rp1aWK1rSOGkiYQKfC5Vyw 1 1 298611 21583 85.1mb 41mb\ngreen open answers_dev4 S0COsGlaSkSWbdgQnkQpww 1 1 2671 3 1.4mb 727.4kb\ngreen open answers_dev5 PIh2r9LaTf2GW-Y5ttwlNw 1 1 38711 452 38.7mb 17.9mb\ngreen open answers_dev6_v1 9ACUvkTrQOGy8dcIpy19GA 1 1 49 0 84.3kb 42.1kb\ngreen open answers_org_pentest_35 pY4rlX7YTOmeaVYGmDRAAQ 1 1 3 0 40.5kb 20.2kb\ngreen open answers_org_staging_35222 azajr74xSd-zd8yDktCn1g 1 1 0 0 416b 208b\ngreen open answers_org_staging_35_v3 Fk9Wff0sTIar_1RsPAhcQA 1 1 18079 1448 5.8mb 2.9mb\ngreen open answers_org_staging_6330 yZ0ikkhTQ5mKQSKq_1MsFQ 1 1 16 0 68.4kb 34.2kb\ngreen open answers_org_staging_6583 4T8JiwxPR2qLGczl5IFvWw 1 1 0 0 462b 231b\ngreen open answers_org_staging_6587 r746EmINRJCVU8U3_WSHZQ 1 1 34 0 84.7kb 42.3kb\ngreen open answers_org_staging_6639 3FrWb7PuSgWMRJQ0nkX2zQ 1 1 15 0 84.5kb 42.2kb\ngreen open answers_org_staging_7794 y8oaeJ2KQtK2VH_z9ltKMQ 1 1 7 0 58kb 29kb\ngreen open answers_pentest UlrWKsyySvG9eMfhrBh6kg 1 1 304276 75292 102.8mb 50.2mb\ngreen open answers_qa_perf_v1 mtPP4u00QNCYzYEw5hU5Vw 1 1 2423 0 742.2kb 371.1kb\ngreen open answers_staging_v2 xFbkM-PESrCt0LlHnIIiKg 1 1 11030606 889771 2.1gb1gb\ngreen open answers_staging_v3 t86StB2zTyKt6YoczQkgzg 1 1 11929362 3589651 2.9gb 1.5gb\n"
From staging:
iex(api@10.2.5.218)2> response.body
"health status index uuid pri rep docs.count docs.deleted store.size pri.store.size\ngreen open answers il41UIeMTKWWdTh1hwDC3g 1 1 5 0 21.6kb 10.8kb\ngreen open answers_dev1_v3 WP9W_71ERCCGN4UJgOV70Q 1 1 454111 187717 163.8mb 91.8mb\ngreen open answers_dev2_v3 kF81UqRMQxG5hqr8KOQMPw 1 1 485951 168523 190mb 93.2mb\ngreen open answers_dev3_v1 Rp1aWK1rSOGkiYQKfC5Vyw 11 298611 21583 85.1mb 41mb\ngreen open answers_dev4 S0COsGlaSkSWbdgQnkQpww 1 1 2671 3 1.4mb 727.4kb\ngreen open answers_dev5 PIh2r9LaTf2GW-Y5ttwlNw 1 1 38720 471 43mb 18.6mb\ngreen open answers_dev6_v1 9ACUvkTrQOGy8dcIpy19GA 1 1 49 0 84.3kb 42.1kb\ngreen open answers_org_pentest_35 pY4rlX7YTOmeaVYGmDRAAQ 1 1 3 0 40.5kb 20.2kb\ngreen open answers_org_staging_35222 azajr74xSd-zd8yDktCn1g 1 1 0 0 416b 208b\ngreen open answers_org_staging_35_v3 Fk9Wff0sTIar_1RsPAhcQA 1 1 18079 1448 5.8mb 2.9mb\ngreen open answers_org_staging_6330 yZ0ikkhTQ5mKQSKq_1MsFQ 1 1 16 0 68.4kb 34.2kb\ngreen open answers_org_staging_6583 4T8JiwxPR2qLGczl5IFvWw 1 1 0 0 462b 231b\ngreen open answers_org_staging_6587 r746EmINRJCVU8U3_WSHZQ 1 1 34 0 84.7kb 42.3kb\ngreen open answers_org_staging_6639 3FrWb7PuSgWMRJQ0nkX2zQ 1 1 15 0 84.5kb 42.2kb\ngreen open answers_org_staging_7794 y8oaeJ2KQtK2VH_z9ltKMQ 1 1 7 0 58kb 29kb\ngreen open answers_pentest UlrWKsyySvG9eMfhrBh6kg 1 1 304276 75292 102.8mb 50.2mb\ngreen open answers_qa_perf_v1 mtPP4u00QNCYzYEw5hU5Vw 1 1 2423 0 742.2kb 371.1kb\ngreen open answers_staging_v2 xFbkM-PESrCt0LlHnIIiKg 1 1 11030606 889771 2.1gb 1gb\ngreen open answers_staging_v3 t86StB2zTyKt6YoczQkgzg 1 1 11929372 3589559 2.9gb 1.5gb\n"
Problems:
- Uglu formating of responses
- Problems with copy pastying long json body
- Solution Kibana, but yes I prefer to use hammer!
For dev2 whe have:
answers_dev2_v3
Steps that I've done:
Read Json from file:
defmodule JsonToMap do
def get_json(filename) do
with {:ok, body} <- File.read(filename),
{:ok, json} <- Poison.decode(body), do: {:ok, json}
end
end
#how to access file from from bastion
template_map = JsonToMap.get_json("/Users/rafaltrojanowski/Projects/Zenloop/backend/apps/domain/priv/indexes/answers.json")
body = %{
"index_patterns": [
"answers",
"answers*"
],
"template": template_map
}
{:ok, response} = Domain.Env.get(:elasticsearch_url) |> URI.merge("/_index_template/answers_template/_mapping?pretty") |> URI.to_string() |> Elastix.HTTP.put()
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Without keywords: