Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
is.go
package main
import (
"crypto/sha256"
"crypto/tls"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"sort"
"strconv"
"strings"
"sync"
"time"
)
type NamespaceRepository struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
Type string `json:"type"`
Status int `json:"status"`
Description string `json:"description"`
Is_automated bool `json:"is_automated"`
Is_private bool `json:"is_private"`
Is_official bool `json:"is_official"`
Star_count int `json:"star_count"`
Pull_count int `json:"pull_count"`
Last_updated string `json:"last_updated"`
}
type QueryRepository struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
Description string `json:"description"`
Is_automated bool `json:"is_automated"`
Is_official bool `json:"is_official"`
Star_count int `json:"star_count"`
Pull_count int `json:"pull_count"`
}
type FileMu struct {
// 'count' is the # of threads acting on this file.
// When its zero then we can remove it from the file map
count int
mu *sync.RWMutex
}
var cache_path = "." + string(os.PathSeparator)
var expire = 10
var used_files map[string]*FileMu = map[string]*FileMu{}
var used_files_mu sync.Mutex
var verbose = 1
var https = &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}}
func debug(v int, format string, args ...interface{}) {
if v > verbose {
return
}
fmt.Printf(format, args...)
}
func cacheGet(url string) []byte {
debug(3, "Get URL: %s\n", url)
h := sha256.New()
h.Write([]byte(url))
name := fmt.Sprintf("%x", h.Sum(nil))
cache_file := cache_path + name
debug(3, " Cache File: %s\n", cache_file)
defer func() {
if r := recover(); r != nil {
debug(0, "Runtime error: %#v\n", r)
}
}()
debug(4, " Locking used_files_mu\n")
used_files_mu.Lock()
// Find file specific lock (create it needed), bump count, and lock it
uf, ok := used_files[name]
if !ok {
debug(4, " Not in used_files map, adding it\n")
uf = &FileMu{
count: 1,
mu: &sync.RWMutex{},
}
used_files[name] = uf
} else {
uf.count++
}
debug(4, " Unlocking used_files_my\n")
used_files_mu.Unlock()
uf.mu.RLock() // Read-lock
bytes, err := ioutil.ReadFile(cache_file)
uf.mu.RUnlock()
// Assume any error means we need to go to the server
if err != nil {
// Block other threads from trying to create the same file
uf.mu.Lock() // write-lock
// First make sure the file is still missing because some
// other thread could have created it while we were locked out
bytes, err = ioutil.ReadFile(cache_file)
// ok, still not there so go ahead and GET it and save it
if err != nil {
if resp, err := https.Get(url); err == nil {
if bytes, err = ioutil.ReadAll(resp.Body); err == nil {
debug(3, " Writing to cache file\n")
err = ioutil.WriteFile(cache_file, bytes, 0644)
if err != nil {
debug(1, " Can't write cache(%s): %s\n",
cache_file, err)
}
} else {
debug(1, " Error in reading http response: %s\n", err)
}
resp.Body.Close()
} else {
debug(1, " Error on GET: %s\n", err)
}
}
uf.mu.Unlock()
} else {
debug(3, " Using cache\n")
}
used_files_mu.Lock()
if uf, ok := used_files[name]; ok {
uf.count--
if uf.count == 0 {
delete(used_files, name)
}
}
used_files_mu.Unlock()
return bytes
}
func checkCache(minutes int) {
for {
files, err := ioutil.ReadDir(cache_path)
if err != nil {
debug(0, "Error reading cache dir: %s\n", err)
// No point continuing if we can't cache stuff
os.Exit(1)
}
for _, f := range files {
// Assume files of 64 chars in length are cache files
if len(f.Name()) != 64 {
continue
}
// Skip any file not older than "minutes" minutes
if f.ModTime().Unix()+int64(minutes*60) >= time.Now().Unix() {
continue
}
used_files_mu.Lock()
// If the file is in our map then lock it
uf, ok := used_files[f.Name()]
if ok {
uf.mu.Lock()
}
os.Remove(f.Name())
if ok {
uf.mu.Unlock()
}
used_files_mu.Unlock()
}
// Look for old files every minute
time.Sleep(time.Duration(1 * int(time.Minute)))
}
}
func main() {
usage := flag.Usage
flag.Usage = func() {
fmt.Println("The Container Search server that can be used to find " +
"container images.")
usage()
}
port := 8080
if os.Getenv("PORT") != "" {
if p, _ := strconv.Atoi(os.Getenv("PORT")); p != 0 {
port = p
}
}
expire := flag.Int("expire", 10, "time (mins) items live in the cache")
flag.StringVar(&cache_path, "path", cache_path, "dir path to the cache")
flag.IntVar(&port, "port", port, "port number for the server to listen on")
flag.IntVar(&verbose, "v", verbose, "verbose/debug level")
flag.Parse()
if cache_path == "" {
cache_path = "."
}
if os.MkdirAll(cache_path, 0644) != nil {
fmt.Printf("unable to create directory: \"%s\"", cache_path)
os.Exit(1)
}
if cache_path[len(cache_path)-1] != os.PathSeparator {
cache_path += string(os.PathSeparator)
}
go checkCache(*expire)
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
bytes, err := ioutil.ReadFile(r.URL.Path[1:])
if err == nil {
w.Write(bytes)
} else {
bytes, err := ioutil.ReadFile("index.html")
if err == nil {
w.Write(bytes)
} else {
w.Write([]byte("404: \"" + r.URL.Path[1:] + "\" not found"))
}
}
})
http.HandleFunc("/search", func(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
w.Header().Set("Content-Type", "application/json")
query := strings.ToLower(r.FormValue("q"))
namespace := strings.ToLower(r.FormValue("n"))
page_size, _ := strconv.Atoi(r.FormValue("s"))
page, _ := strconv.Atoi(r.FormValue("p"))
order := r.FormValue("r")
official := -1
if o := r.FormValue("o"); len(o) != 0 {
if o == "1" {
official = 1
} else {
official = 0
}
}
automated := -1
if a := r.FormValue("a"); len(a) != 0 {
if a == "1" {
automated = 1
} else {
automated = 0
}
}
if page_size < 1 {
page_size = 100
}
if page < 0 {
page = 0
}
if len(namespace) != 0 {
var repos []NamespaceRepository
start := page * page_size
end := start + page_size
if !(official == 0 && namespace == "library") && (official != 1 || namespace == "library") {
is_official := namespace == "library"
for i := 1; ; i++ {
bytes := cacheGet("https://hub.docker.com/v2/repositories/" + namespace + "?page_size=1000&page=" + strconv.Itoa(i))
if len(bytes) > 0 {
var data map[string]interface{}
err := json.Unmarshal(bytes, &data)
if err == nil {
if data["detail"] != nil && data["detail"].(string) == "Not found" {
break
}
results := data["results"].([]interface{})
for e := 0; e < len(results); e++ {
result := results[e].(map[string]interface{})
name := ""
if r := result["name"]; r != nil {
name = r.(string)
}
description := ""
if r := result["description"]; r != nil {
description = r.(string)
}
if len(query) > 0 && !(strings.Contains(strings.ToLower(name), query) || strings.Contains(strings.ToLower(description), query)) {
continue
}
is_automated := false
if r := result["is_automated"]; r != nil {
is_automated = r.(bool)
}
if (automated == 0 && is_automated) || (automated == 1 && !is_automated) {
continue
}
is_private := false
if r := result["is_private"]; r != nil {
is_private = r.(bool)
}
namespace := ""
if r := result["namespace"]; r != nil {
namespace = r.(string)
}
type_ := ""
if r := result["repository_type"]; r != nil {
type_ = r.(string)
}
status := 0
if r := result["status"]; r != nil {
status = int(r.(float64))
}
star_count := 0
if r := result["star_count"]; r != nil {
star_count = int(r.(float64))
}
pull_count := 0
if r := result["pull_count"]; r != nil {
pull_count = int(r.(float64))
}
last_updated := ""
if r := result["last_updated"]; r != nil {
last_updated = r.(string)
}
repos = append(repos, NamespaceRepository{name, namespace, type_, status, description, is_automated, is_private, is_official, star_count, pull_count, last_updated})
}
} else {
break
}
} else {
break
}
}
switch order {
case "star_count":
sort.Slice(repos[:], func(a, b int) bool {
return repos[a].Star_count > repos[b].Star_count
})
break
case "-star_count":
sort.Slice(repos[:], func(a, b int) bool {
return repos[a].Star_count < repos[b].Star_count
})
break
case "pull_count":
sort.Slice(repos[:], func(a, b int) bool {
return repos[a].Pull_count > repos[b].Pull_count
})
break
case "-pull_count":
sort.Slice(repos[:], func(a, b int) bool {
return repos[a].Pull_count < repos[b].Pull_count
})
break
}
}
response := "{\"count\":" + strconv.Itoa(len(repos))
if start >= len(repos) {
repos = repos[:0]
} else {
if end > len(repos) {
end = len(repos)
}
repos = repos[start:end]
}
results, err := json.Marshal(repos)
if err == nil {
response += ",\"results\":" + string(results)
} else {
response += ",\"results\":\"[]"
}
w.Write([]byte(response + "}\n"))
} else if len(query) != 0 {
var repos []QueryRepository
start := page * page_size
offset := int(start % 100)
count := 0
query = url.QueryEscape(query)
// swap negatives for the docker hub server
if len(order) > 0 {
if order[0:1] == "-" {
order = "&ordering=" + order[1:]
} else {
order = "&ordering=-" + order
}
}
// -1 = either, 0 = not official, 1 = official
if official != -1 {
if official == 1 {
order += "&is_official=1"
} else {
order += "&is_official=0"
}
}
if automated != -1 {
if automated == 1 {
order += "&is_automated=1"
} else {
order += "&is_automated=0"
}
}
query_loop:
for i := int(start/100) + 1; len(repos) < page_size; i++ {
bytes := cacheGet("https://hub.docker.com/v2/search/repositories/" + "?page_size=100&page=" + strconv.Itoa(i) + "&query=" + query + order)
if len(bytes) > 0 {
var data map[string]interface{}
err := json.Unmarshal(bytes, &data)
if err == nil {
if data["message"] != nil || data["text"] != nil {
break
}
if count == 0 {
count = int(data["count"].(float64))
}
results := data["results"].([]interface{})
for e := offset; e < len(results); e++ {
offset = 0
if len(repos)+1 > page_size {
break query_loop
}
result := results[e].(map[string]interface{})
name := ""
if r := result["repo_name"]; r != nil {
name = r.(string)
}
namespace := ""
if s := strings.Index(name, "/"); s > -1 {
namespace = name[0:s]
name = name[s+1:]
} else {
namespace = "library"
}
description := ""
if r := result["short_description"]; r != nil {
description = r.(string)
}
is_automated := false
if r := result["is_automated"]; r != nil {
is_automated = r.(bool)
}
is_official := false
if r := result["is_official"]; r != nil {
is_official = r.(bool)
}
star_count := 0
if r := result["star_count"]; r != nil {
star_count = int(r.(float64))
}
pull_count := 0
if r := result["pull_count"]; r != nil {
pull_count = int(r.(float64))
}
repos = append(repos, QueryRepository{name, namespace, description, is_automated, is_official, star_count, pull_count})
}
} else {
break
}
} else {
break
}
}
response := "{\"count\":" + strconv.Itoa(count)
results, err := json.Marshal(repos)
if err == nil {
response += ",\"results\":" + string(results)
}
w.Write([]byte(response + "}\n"))
} else {
w.WriteHeader(400)
w.Write([]byte("{\"error\":{\"message\":\"Incorrect query parameters\"}}\n"))
}
})
// TODO add -v to control debugging info printed
debug(0, "Listening on port %d\n", port)
http.ListenAndServe(":"+strconv.Itoa(port), nil)
// TODO split into files
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment