Skip to content

Instantly share code, notes, and snippets.

@jamesu
Created October 22, 2021 01:07
Show Gist options
  • Save jamesu/e684f93e38cd1f84b0d679edddfb80a7 to your computer and use it in GitHub Desktop.
Save jamesu/e684f93e38cd1f84b0d679edddfb80a7 to your computer and use it in GitHub Desktop.
Experiment getting gitlab runner to run jobs without gitlab
package main
import (
"fmt"
"net/http"
"log"
"os"
"io/ioutil"
"encoding/json"
b64 "encoding/base64"
"reflect"
mux "github.com/gorilla/mux"
"strconv"
//"github.com/ghodss/yaml"
)
// Gitea API structs
type GiteaProject struct {
Id int `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
}
// Gitlab API structs
type GitlabFeatureFlags struct {
Variables bool `json: "variables"`
Image bool `json: "image"`
Services bool `json: "services"`
Artifacts bool `json: "artifacts"`
Cache bool `json: "cache"`
Shared bool `json: "shared"`
UploadMultipleArtifacts bool `json: "upload_multiple_artifacts"`
UploadRawArtifacts bool `json: "upload_raw_artifacts"`
Session bool `json: "session"`
Terminal bool `json: "terminal"`
Refspecs bool `json: "refspecs"`
Masking bool `json: "masking"`
Proxy bool `json: "proxy"`
RawVariables bool `json: "raw_variables"`
ArtifactsExclude bool `json: "artifacts_exclude"`
MultiBuildSteps bool `json: "multi_build_steps"`
TraceReset bool `json: "trace_reset"`
TraceChecksum bool `json: "trace_checksum"`
TraceSize bool `json: "trace_size"`
VaultSecrets bool `json: "vault_secrets"`
Cancelable bool `json: "cancelable"`
ReturnExitCode bool `json: "return_exit_code"`
}
type GitlabFeatureInfo struct {
TraceSections bool `json: "trac_sections"`
FailureReasons string `json: "failure_reasons"`
}
type GitlabConfig struct {
Gpus int `json: "gpus"`
}
type GitlabVersionInfo struct {
Name string `json:"name,omitempty"`
Version string `json:"version,omitempty"`
Revision string `json:"revision,omitempty"`
Platform string `json:"platform,omitempty"`
Architecture string `json:"architecture,omitempty"`
Executor string `json:"executor,omitempty"`
Shell string `json:"shell,omitempty"`
Features GitlabFeatureFlags `json:"features,omitempty"`
Config GitlabConfig `json:"config,omitempty"`
}
type GitlabOutputInfo struct {
Checksum string `json: "checksum"`
Bytesize int `json: "bytesize"`
}
type GitlabRequestQuery struct {
Info GitlabVersionInfo `json: "info"`
Token string `json: "token"`
LastUpdate string `json:"last_update" `
// If something happened
State string `json:"state,omitempty" `
Checksum string `json:"checksum,omitempty" `
Output GitlabOutputInfo `json:"output,omitempty" `
}
// NewSomething create new instance of Something
func NewGitlabFeatureFlags() GitlabFeatureFlags {
feat := GitlabFeatureFlags{}
feat.Variables = true
feat.Image = false
feat.Services = false
feat.Artifacts = true
feat.Cache = true
feat.Shared = true
feat.UploadMultipleArtifacts = true
feat.UploadRawArtifacts = true
feat.Session = true
feat.Terminal = true
feat.Refspecs = true
feat.Masking = true
feat.Proxy = false
feat.RawVariables = true
feat.ArtifactsExclude = true
feat.MultiBuildSteps = true
feat.TraceReset = true
feat.TraceChecksum = true
feat.TraceSize = true
feat.VaultSecrets = true
feat.Cancelable = true
feat.ReturnExitCode = true
return feat
}
type GitlabJobInfo struct {
Id int `json: "id"`
Name string `json: "name"`
Stage string `json: "stag"`
ProjectId int `json: "project_id"`
ProjectName string `json: "project_name"`
}
type GitlabGitInfo struct {
RepoUrl int `json: "repo_url"`
Ref int `json: "ref"`
Sha int `json: "sha"`
BeforeSha int `json: "before_sha"`
RefType int `json: "ref_type"`
RefSpecs []string `json: "refspecs"`
Depth int `json: "depth"`
}
type GitlabRunnerInfo struct {
Timeout int `json: "timeout"`
RunnerSessionUrl string `json: "runner_session_url"`
}
type GitlabVariable struct {
Key string `json: "key"`
Value string `json: "value"`
Public bool `json: "public"`
Masked bool `json: "masked"`
}
type GitlabStep struct {
Name string `json: "name"`
Script []string `json: "script"`
Timeout int `json: "timeout"`
When string `json: "when"`
AllowFailure bool `json: "allow_failure"`
}
type GitlabCredential struct {
Type string `json: "type"`
Url string `json: "url"`
Username string `json: "username"`
Password string `json: "password"`
}
type GitlabDependencyArtifactsFile struct {
Filename string `json:"filename"`
Size int64 `json:"size"`
}
type GitlabDependency struct {
Id int `json: "id"`
Name string `json: "name"`
Token string `json: "token"`
ArtifactsFile GitlabDependencyArtifactsFile `json:"artifacts_file"`
}
type GitlabCacheKey struct {
Files []string `json: "files"`
Key string
}
func (p GitlabCacheKey) MarshalJSON() ([]byte, error) {
if (p.Key != "") {
return json.Marshal(p.Key)
} else {
return json.Marshal(struct {
Files []string `json:"files"`
}{
Files: p.Files,
})
}
}
func (p GitlabCacheKey) UnmarshalJSON(b []byte) error {
var stuff string
err := json.Unmarshal(b, &stuff)
if err != nil {
print("GitlabCacheKey (prob struct) err=")
print(err)
err := json.Unmarshal(b, &p)
return err
}
p.Key = stuff
return nil
}
type GitlabCacheEntry struct {
Key GitlabCacheKey `json: "key"`
Untracked string `json: "untracked"`
Paths string `json: "paths"`
When string `json: "when"`
Policy string `json: "policy"`
}
type GitlabImagePort struct {
Number int `json:"number,omitempty"`
Protocol string `json:"protocol,omitempty"`
Name string `json:"name,omitempty"`
}
type GitlabImage struct {
Name string `json:"name"`
Alias string `json:"alias,omitempty"`
Command []string `json:"command,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"`
Ports []GitlabImagePort `json:"ports,omitempty"`
}
type GitlabArtifact struct {
Name string `json:"name"`
Untracked bool `json:"untracked"`
Paths []string `json:"paths"`
Exclude []string `json:"exclude"`
When []string `json:"when"`
Type string `json:"artifact_type"`
Format string `json:"artifact_format"`
ExpireIn string `json:"expire_in"`
}
type GitlabSecrets map[string]GitlabSecret
type GitlabSecret struct {
Vault *GitlabVaultSecret `json:"vault,omitempty"`
File *bool `json:"file,omitempty"`
}
type GitlabVaultSecret struct {
Server GitlabVaultServer `json:"server"`
Engine GitlabVaultEngine `json:"engine"`
Path string `json:"path"`
Field string `json:"field"`
}
type GitlabVaultServer struct {
URL string `json:"url"`
Auth GitlabVaultAuth `json:"auth"`
}
type GitlabVaultAuthData map[string]interface{}
type GitlabVaultAuth struct {
Name string `json:"name"`
Path string `json:"path"`
Data GitlabVaultAuthData `json:"data"`
}
type GitlabVaultEngine struct {
Name string `json:"name"`
Path string `json:"path"`
}
type GitlabJobRequest struct {
Info GitlabVersionInfo `json:"info,omitempty"`
Token string `json:"token,omitempty"`
LastUpdate string `json:"last_update,omitempty"`
Session *GitlabSessionInfo `json:"session,omitempty"`
}
type GitlabSessionInfo struct {
URL string `json:"url,omitempty"`
Certificate string `json:"certificate,omitempty"`
Authorization string `json:"authorization,omitempty"`
}
type GitlabJobResponse struct {
Id int `json: "id"`
Token string `json: "token"`
AllowGitFetch bool `json: "allow_git_fetch"`
JobInfo GitlabJobInfo `json: "job_info"`
GitInfo GitlabGitInfo `json: "git_info"`
RunnerInfo GitlabRunnerInfo `json: "runner_info"`
Variables []GitlabVariable `json: "variables"`
Steps []GitlabStep `json: "steps"`
Image GitlabImage `json: "image"`
Services []string `json: "services"`
Artifacts []GitlabArtifact `json: "artifacts"`
Cache []GitlabCacheEntry `json: "cache"`
Credentials []GitlabCredential `json: "credentials"`
Dependencies []GitlabDependency `json: "dependencies"`
Features GitlabFeatureInfo `json: "features"`
Secrets GitlabSecrets `json:"secrets,omitempty"`
}
type GitlabRegisterRunnerParameters struct {
Description string `json:"description,omitempty"`
Tags string `json:"tag_list,omitempty"`
RunUntagged bool `json:"run_untagged"`
Locked bool `json:"locked"`
AccessLevel string `json:"access_level,omitempty"`
MaximumTimeout int `json:"maximum_timeout,omitempty"`
Active bool `json:"active"`
}
type GitlabRegisterRunnerRequest struct {
GitlabRegisterRunnerParameters
Info GitlabVersionInfo `json:"info,omitempty"`
Token string `json:"token,omitempty"`
}
type GitlabTokenQuery struct {
Token string `json:"token,omitempty"`
}
type GitlabRegistrationResponse struct {
Message map[string]string `json: "message"`
}
func main() {
var JobSent bool
TestJob := GitlabJobResponse{}
TestJobWorkerToken := "TEST_WORKER_TOKEN"
TestWorkerRegistered := false
// Build folder should be of form: project-ProjectID-ProjectRunnerID
CurrentJobId := 100
TestJob.Id = 100
TestJob.Token = "TEST_JOB_TOKEN"
TestJob.AllowGitFetch = false
TestJob.Variables = []GitlabVariable{
GitlabVariable{ Key: "CI_PIPELINE_ID", Value: "", Public: true, Masked: false },
GitlabVariable{ Key: "CI_PIPELINE_URL", Value: "http://lappo.local:8020/mango/project1/-/pipelines/1", Public: true, Masked: false },
GitlabVariable{ Key: "CI_JOB_URL", Value: "http://lappo.local:8020/mango/project1/-/jobs/" + string(CurrentJobId), Public: true, Masked: false },
GitlabVariable{ Key: "CI_JOB_TOKEN", Value: "AAA", Public: true, Masked: false },
GitlabVariable{ Key: "CI_REPOSITORY_URL", Value: "AAA", Public: true, Masked: false },
GitlabVariable{ Key: "CI_JOB_NAME", Value: "build-job", Public: true, Masked: false },
GitlabVariable{ Key: "CI_JOB_STAGE", Value: "build", Public: true, Masked: false },
GitlabVariable{ Key: "CI_BUILD_NAME", Value: "build-job", Public: true, Masked: false },
GitlabVariable{ Key: "CI", Value: "true", Public: true, Masked: false },
GitlabVariable{ Key: "GITLAB_CI", Value: "true", Public: true, Masked: false } }
TestJob.JobInfo.Id = CurrentJobId
TestJob.JobInfo.Name = "test_job"
TestJob.JobInfo.ProjectId = 1
TestJob.JobInfo.ProjectName = "project1"
TestJob.JobInfo.Stage = "test_job"
TestJob.GitInfo = GitlabGitInfo{}
TestJob.RunnerInfo = GitlabRunnerInfo{ Timeout: 10000 }
TestJob.Steps = []GitlabStep{
GitlabStep{ Name: "script", Script: []string{"echo \"Helllo world!\""}, Timeout: 3600, When: "on_success", AllowFailure: false },
}
router := mux.NewRouter()
// Jobs
router.HandleFunc("/api/v4/runners", func(w http.ResponseWriter, r *http.Request) {
// Should return a list of runners OR register one
w.Header().Set("Content-Type", "application/json")
switch r.Method {
//case http.MethodGet:
case http.MethodPost:
var reg_params GitlabRegisterRunnerRequest
json.NewDecoder(r.Body).Decode(&reg_params)
fmt.Printf("REG REQUEST: %+v\n", reg_params)
response := GitlabTokenQuery{}
response.Token = TestJobWorkerToken
w.WriteHeader(http.StatusCreated)
json.NewEncoder(w).Encode(response)
TestWorkerRegistered = true
case http.MethodDelete:
var token_params GitlabTokenQuery
json.NewDecoder(r.Body).Decode(&token_params)
fmt.Printf("UNREG REQUEST: %+v\n", token_params)
if TestWorkerRegistered {
w.WriteHeader(http.StatusNoContent)
} else {
w.WriteHeader(http.StatusNoContent)
}
TestWorkerRegistered = false
default:
http.Error(w, "{}", http.StatusBadRequest)
}
})
router.HandleFunc("/api/v4/jobs/request", func(w http.ResponseWriter, r *http.Request) {
// Posts basic feature info with no state usually
// Should return 200 OR
// 201 Created & GitlabJobResponse
// Gitlab-Ci-Builds-Polling: yes
fmt.Println("REQ JOB!")
w.Header().Set("Content-Type", "application/json")
switch r.Method {
case http.MethodPost:
var status GitlabJobRequest
json.NewDecoder(r.Body).Decode(&status)
if status.Token != TestJobWorkerToken {
fmt.Println("Worker submitted bad token!")
w.WriteHeader(http.StatusBadRequest)
return;
}
if !JobSent {
fmt.Println("Sending example job")
w.WriteHeader(http.StatusCreated)
json.NewEncoder(w).Encode(TestJob)
} else {
fmt.Println("Job submitted already, no more jobs")
w.WriteHeader(http.StatusNoContent)
}
JobSent = true
default:
http.Error(w, "Invalid", http.StatusMethodNotAllowed)
}
})
router.HandleFunc("/api/v4/jobs/runner/verify", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch r.Method {
case http.MethodPost:
if !JobSent {
json.NewEncoder(w).Encode(TestJob)
} else {
w.WriteHeader(200)
}
default:
http.Error(w, "Invalid", http.StatusMethodNotAllowed)
}
})
router.HandleFunc("/api/v4/jobs/{id}", func(w http.ResponseWriter, r *http.Request) {
// Should normally return 200
// Basically updates state of runner
vars := mux.Vars(r)
jobId, _ := strconv.Atoi(vars["id"])
if jobId != CurrentJobId {
w.WriteHeader(http.StatusNotFound)
return
}
switch r.Method {
case http.MethodPut:
bytes, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, "Invalid body", http.StatusMethodNotAllowed)
return
}
os.Stdout.Write(bytes[:])
w.WriteHeader(http.StatusOK)
default:
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
}
})
router.HandleFunc("/api/v4/jobs/{id}/trace", func(w http.ResponseWriter, r *http.Request) {
// Requires: job-token
// TODO: check this
//jobToken := r.Header.Get("Job-Token")
vars := mux.Vars(r)
jobId, _ := strconv.Atoi(vars["id"])
if jobId != CurrentJobId {
w.WriteHeader(http.StatusNotFound)
return
}
switch r.Method {
case http.MethodPatch:
bytes, err := ioutil.ReadAll(r.Body)
if err != nil {
fmt.Printf("error=%+v\n", err);
http.Error(w, "Invalid body", http.StatusMethodNotAllowed)
return
}
os.Stdout.Write(bytes[:])
w.WriteHeader(http.StatusOK)
default:
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
}
//
})
// Build artifacts
router.HandleFunc("/api/v4/jobs/{id}/artifacts", func(w http.ResponseWriter, r *http.Request) {
// TODO
})
router.HandleFunc("/api/v4/jobs/{id}/artifacts/{artifact_id}", func(w http.ResponseWriter, r *http.Request) {
// TODO
})
router.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {
var body []byte = nil
finished := make(chan bool)
poll_gitea := func () {
client := &http.Client{}
req, err := http.NewRequest("GET", "http://192.168.2.112:8081/api/v1/repos/mango/hello-woodpecker/contents/.woodpecker.yml", nil)
req.Header.Add("Authorization", "token " + "f67a78c84db0eff0c7aced2f676a039b06f712ca")
if err != nil {
log.Print(err)
os.Exit(1)
}
// token=f67a78c84db0eff0c7aced2f676a039b06f712ca
resp, err := client.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
ibody, _ := ioutil.ReadAll(resp.Body)
body = ibody
finished <- true
}
go poll_gitea()
<- finished
var objmap map[string]json.RawMessage
err := json.Unmarshal(body, &objmap)
if err != nil {
panic(err)
}
var filedata string = ""
json.Unmarshal(objmap["content"], &filedata)
fdatas, _ := b64.StdEncoding.DecodeString(filedata)
fmt.Fprintf(w, "Response %s\n", fdatas)
})
router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
var body []byte = nil
finished := make(chan bool)
poll_gitea := func () {
client := &http.Client{}
req, err := http.NewRequest("GET", "http://192.168.2.112:8081/api/v1/repos/search", nil)
req.Header.Add("Authorization", "token " + "f67a78c84db0eff0c7aced2f676a039b06f712ca")
if err != nil {
log.Print(err)
os.Exit(1)
}
// token=f67a78c84db0eff0c7aced2f676a039b06f712ca
resp, err := client.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
ibody, _ := ioutil.ReadAll(resp.Body)
body = ibody
finished <- true
}
go poll_gitea()
<- finished
var objmap map[string]json.RawMessage
err := json.Unmarshal(body, &objmap)
if err != nil {
panic(err)
}
keys := reflect.ValueOf(objmap).MapKeys()
fmt.Printf("%+v\n", keys)
var plist []GiteaProject = make([]GiteaProject,0);
json.Unmarshal(objmap["data"], &plist)
fmt.Fprintf(w, "<table><thead><tr><th>id</th><th>tag</th><th>name</th></tr></thead><tbody>")
for _, proj := range plist {
fmt.Printf("%+v\n", proj)
fmt.Fprintf(w, "<tr><td>%d</td><td>%s</td><td>%s</td></tr>\n", proj.Id, proj.Name, proj.FullName)
}
fmt.Fprintf(w, "</tbody></table>")
fmt.Fprintf(w, "Response %s\n", body)
})
http.Handle("/", router)
http.ListenAndServe("0.0.0.0:8020", nil)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment