Skip to content

Instantly share code, notes, and snippets.

@whizzmler
Last active May 10, 2016 21:15
Show Gist options
  • Save whizzmler/5f1f0a58683b3eb5efffaab47683ad27 to your computer and use it in GitHub Desktop.
Save whizzmler/5f1f0a58683b3eb5efffaab47683ad27 to your computer and use it in GitHub Desktop.
Model or Ensemble?
{
"name": "Model or ensemble",
"description": "Select the best option for modeling a source: a model or an ensemble?",
"inputs": [
{
"name": "input-source-id",
"type": "source-id",
"description": "Source for training/test the model and ensemble"
}
],
"outputs": [
{
"name": "result",
"type": "string",
"description": "Either a model or ensemble identifier"
}
]
}
;; Functions for creating the two dataset parts
;; and the model and ensemble from the training set.
(define (sample-dataset ds-id rate oob)
(create-and-wait-dataset {"sample_rate" rate
"origin_dataset" ds-id
"out_of_bag" oob
"seed" "whizzml-example"}))
(define (split-dataset ds-id rate)
(list (sample-dataset ds-id rate false)
(sample-dataset ds-id rate true)))
(define (make-model ds-id)
(create-and-wait-model {"dataset" ds-id}))
(define (make-ensemble ds-id size)
(create-and-wait-ensemble {"dataset" ds-id
"number_of_models" size}))
;; Functions for evaluating model and ensemble
;; using the test set, and to extract f-measure from
;; the evaluation results
(define (evaluate-model model-id ds-id)
(create-and-wait-evaluation {"model" model-id
"dataset" ds-id}))
(define (evaluate-ensemble model-id ds-id)
(create-and-wait-evaluation {"ensemble" model-id
"dataset" ds-id}))
(define (f-measure ev-id)
(get-in (fetch ev-id) ["result" "model" "average_f_measure"]))
;; Function encapsulating the full workflow
(define (model-or-ensemble src-id)
(let (ds-id (create-and-wait-dataset {"source" src-id})
;; ^ full dataset
ids (split-dataset ds-id 0.8) ;; split it 80/20
train-id (nth ids 0) ;; the 80% for training
test-id (nth ids 1) ;; and 20% for evaluations
m-id (make-model train-id) ;; create a model
e-id (make-ensemble train-id 15) ;; and an ensemble
m-f (f-measure (evaluate-model m-id test-id)) ;; evaluate
e-f (f-measure (evaluate-ensemble e-id test-id)))
(log-info "model f " m-f " / ensemble f " e-f)
(if (> m-f e-f) m-id e-id)))
;; Compute the result of the script execution
;; - Inputs: [{"name": "input-source-id", "type": "source-id"}]
;; - Outputs: [{"name": "result", "type": "resource-id"}]
(define result (model-or-ensemble input-source-id))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment