Skip to content

Instantly share code, notes, and snippets.

@vagetablechicken
Last active July 24, 2023 04:28
Show Gist options
  • Save vagetablechicken/617aacc3c1507467664da6902cc9f47b to your computer and use it in GitHub Desktop.
Save vagetablechicken/617aacc3c1507467664da6902cc9f47b to your computer and use it in GitHub Desktop.
Useful scripts

scripts about cmd, github gh cli ...

# move or remove them before you merge, use this to get related files
git xxx 2>&1|grep -E '^\s'|cut -f2-|xargs -I {} echo "{}"
#!/bin/bash
set -ex
# /Users/ark
PWD=$(pwd)
WORK_DIR="$PWD"/Code/OpenMLDB
pushd "$WORK_DIR"
time=$(date +%Y-%m-%d\ %H:%M:%S)
echo "start rebuilding at $time, dir $PWD"
make clean
make SQL_JAVASDK_ENABLE=ON
echo "rebuild succeed"
popd
# set this to crontab
# e.g. 0 0 * * * sh /Users/ark/Code/OpenMLDB/rebuild.sh >> /Users/ark/Code/OpenMLDB/rebuild.log 2>&1
#. will do it every day (at 00:00)
# test crontab
# if you use 'xx yy * * *', crontab will run it next day, only all * works(every minute), but it's dangerous.
# so make a crontab env:
# 1.crontab -e, then add '* * * * * /usr/bin/env > /Users/ark/cron-env', cron-env will update every minute.
# 2.create a exec 'run-as-cron':
# ```
# !/bin/sh
#
# . "$1"
# exec /usr/bin/env -i "$SHELL" -c ". $1; $2"
# ```
# 3. chmod +x, then run it. usage: run-as-cron <cron-environment> <command>
# 4. ./run-as-cron cron-env "sh xxx/rebuild.sh > xxx/rebuild.log 2>&1"
ps axu | grep xx | awk '{print $2}' | xargs kill -9
netstat -ltnup
import re
import os
import subprocess
import sys
import time
# output = subprocess.check_output(['gh','-R','4paradigm/OpenMLDB', 'pr', 'checks', '3312']) # -R 4paradigm/OpenMLDB pr checks 3312'])
# print(output)
# result = subprocess.run(['gh','-R','4paradigm/OpenMLDB', 'pr', 'checks', '3312'])#, shell=True)
from subprocess import Popen, PIPE
repo = "4paradigm/OpenMLDB"
assert len(sys.argv) >= 2, "input pr is needed"
pr_list = sys.argv[1]
if len(sys.argv) > 2:
repo = sys.argv[2]
if not pr_list:
print("no pr input. exit.")
exit(-1)
prs = []
if pr_list == 'me':
p = Popen(f"gh pr list -A vagetablechicken -R {repo}".split(" "), stdin=PIPE, stdout=PIPE, stderr=PIPE)
(
output,
err,
) = p.communicate() # b"input data that is passed to subprocess' stdin")
rc = p.returncode
# print(output)
# split jobs result
result = output.decode("utf-8")
prs = [pr.split("\t")[0] for pr in result.split("\n") if pr]
else:
# pr -> prs, split by ,
prs = pr_list.split(",")
watchs = [
"cpp",
"macos-cpp",
"coverage",
"python-sdk",
"java-sdk",
"openmldb-test-python",
"go-sdk",
"openmldb-tool-test",
]
status = []
def check_pr(pr):
# print(f'check {pr} in repo {repo}')
pr_status = []
def get_job_status():
p = Popen(
["gh", "-R", repo, "pr", "checks", pr], stdin=PIPE, stdout=PIPE, stderr=PIPE
)
(
output,
err,
) = p.communicate() # b"input data that is passed to subprocess' stdin")
rc = p.returncode
# print(output)
# split jobs result
result = output.decode("utf-8")
jobs = [job.split("\t") for job in result.split("\n")]
# print(jobs)
return jobs
jobs = get_job_status()
# add a fail to test
# jobs.append(['cpp', 'fail', '', 'https://github.com/4paradigm/OpenMLDB/actions/runs/5220800424/jobs/9425957322'])
for job in jobs:
if job[0] not in watchs:
continue
# print('check', job)
if job[1] == "fail":
# parse run id
x = re.search("runs/[0-9]+", job[3])
if not x:
print("no run id?", job[3])
else:
id = x.group(0)
id = id[5:]
print("rerun", id)
try:
subprocess.check_output(
f"gh -R 4paradigm/OpenMLDB run rerun --failed {id}", shell=True
)
except subprocess.CalledProcessError as e:
print(e.output)
pr_status.append((job[0], job[1]))
all_pass = False
if all(s == "pass" for _, s in pr_status):
print(f"pr {pr} all pass")
all_pass = True
else:
print(f"pr {pr} not all pass:", pr_status)
return all_pass, pr_status
print(f"start watch {prs} in repo {repo}, watchs {watchs}")
# pr done if all pass 3 times
pr_st = { pr: 0 for pr in prs }
while True:
for pr, succ in pr_st.items():
if succ > 2:
continue
done, _ = check_pr(pr)
if done:
pr_st[pr] += 1
if all(succ > 2 for succ in pr_st.values()):
print("all pr pass 3 times. exit.")
exit(0)
print(pr_st)
time.sleep(5 * 60)
assert False, "?"
@vagetablechicken
Copy link
Author

vagetablechicken commented Dec 28, 2021

rebuild.sh needs path env? new make will get error cmake: No such file or directory
solution: export PATH(if it contains cmake bin path, skip add) before make
PATH exists but not export by default, so we should export it. So the root Makefile can use $PATH, you can echo $PATH in Makefile to test.

@vagetablechicken
Copy link
Author

vagetablechicken commented Jun 9, 2023

gh run list -w cicd and other workflow, watch all? python is better

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment