Instantly share code, notes, and snippets.

Embed
What would you like to do?
Flights parser
(ns parsy.background.core
(:require-macros [cljs.core.async.macros :refer [go go-loop]])
(:require [cljs.core.async :refer [chan <! >! merge]]
[cljs-time.core :refer [days plus after?]]
[cljs-time.format :refer [formatter parse unparse]]))
(enable-console-print!)
; Map of tab-id => chan
(def waiting (atom {}))
; Puts received message to waiting channel:
(.. js/chrome -runtime -onMessage (addListener
#(go (let [result (js->clj %1 :keywordize-keys true)
{:keys [tab]} (js->clj %2 :keywordize-keys true)
waiter (get @waiting (:id tab))]
(>! waiter result)))))
(defn run-script
[{:keys [id]}]
(let [result (chan)]
(.. js/chrome -tabs (executeScript id #js {:file "content/main.js"}))
(swap! waiting assoc id result)
result))
(defn open-tab
[url]
(let [done (chan)]
(.. js/chrome -tabs (create #js {:url url}
#(go (>! done %))))
done))
(defn make-url
[id-from id-to date]
(str "https://avia.yandex.ru/search/?fromId=" id-from
"&toId=" id-to
"&when=" date
"&return_date=&oneway=2&adult_seats=1&children_seats=0&infant_seats=0&klass=economy"))
(defn days-range
[date-from date-to]
(let [input-format (formatter "MM.dd")
date-from (parse input-format date-from)
date-to (parse input-format date-to)
get-range (fn get-range [start]
(if (after? start date-to)
nil
(cons start (lazy-seq (get-range (plus start (days 1)))))))]
(get-range date-from)))
(defn get-flights
[id-from id-to date]
(go (let [search-format (formatter "dd+MMM")
tab (<! (open-tab (make-url id-from id-to (unparse search-format date))))
{:keys [id]} (js->clj tab :keywordize-keys true)
{:keys [flights]} (<! (run-script tab))]
(.. js/chrome -tabs (remove id))
(map #(assoc % :date date) flights))))
(defn concat-flights
[flights]
(go-loop [[flight & flights] flights
result []]
(if flights
(recur flights (concat result (<! flight)))
result)))
(defn present
[prices]
(let [present-format (formatter "MM.dd")]
(go (->> (<! prices)
(map #(update % :date (fn [date] (unparse present-format date))))
clj->js
(.table js/console)))))
(defn run
[id-from id-to date-from date-to]
(->> (days-range date-from date-to)
(map #(get-flights id-from id-to %))
concat-flights
present))
(set! (.-run js/window) run)
(ns parsy.content.core
(:require-macros [cljs.core.async.macros :refer [go-loop go]])
(:require [cljs.core.async :refer [timeout <!]]
[clojure.string :as string]
[dommy.core :refer-macros [sel1 sel]]))
(enable-console-print!)
(defn text
[line selector]
(when-let [el (sel1 line selector)]
(string/replace (.-textContent el) #"&nbsp;" "")))
(defn price
[val]
(js/parseInt (string/replace val #"[^\d]" "") 10))
(defn arrival
[val]
(string/join "" (concat (take 5 val) [" "] (drop 5 val))))
(defn get-flights
[]
(for [line (sel ".flight_list__forward")]
{:company (text line ".flight_list__company-names")
:departure (text line ".flight_list__departure-time")
:duration (text line ".flight_list__flight-duration")
:arrival (arrival (text line ".flight_list__arrival-time"))
:price (price (text line ".price_kb"))}))
(defn ready?
[]
(sel1 ".flight_list"))
(go-loop []
(if (ready?)
(.. js/chrome -runtime (sendMessage #js {:status :ok
:flights (clj->js (get-flights))}))
(do (<! (timeout 500))
(recur))))
{
"name": "parsy",
"version": "0.1",
"background": {
"scripts": [
"background/main.js"
],
"persistent": false
},
"permissions": [
"tabs",
"http://*/*",
"https://*/*"
],
"manifest_version": 2
}
@franciscocpg

This comment has been minimized.

franciscocpg commented Jan 8, 2016

How can I compile/run this code?

@nvbn

This comment has been minimized.

Owner

nvbn commented Apr 14, 2017

@franciscocpg It's code from article - https://nvbn.github.io/2015/07/17/flight-prices/
For running it you need to create cljs project, compile cljs to js and load it as a chrome extension.

Also yandex avia was changed a bit, so it's not working anymore.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment