| // Compile with | |
| // gcc -o /etc/bin/autossh autossh.c | |
| #include <stdio.h> | |
| int main(int argc, char *argv[]) { | |
| while(1) { | |
| sleep(60); | |
| printf(".\n"); | |
| } | |
| } |
| # Latest git commit hash | |
| git log --pretty=format:'%h' -n1 | |
| # Search for a gem on rubygem servers | |
| gem search -r eventmachine | |
| # List files and directories and sort on their sizes | |
| du -hs * | sort -h | |
| # Show how much space the current directory uses | |
| du -hs | |
| # Show overall disc usage stats | |
| df -h | |
| # Grep with extended regular expression | |
| # http://www.cyberciti.biz/faq/grep-regular-expressions/ | |
| grep -E "slot=(8,9,10)" | |
| # Return only the matched content instead of the whole line - helpful for minified javascript | |
| grep -ioR '\./assets.*swf?' ./ | |
| # -l - return only the filename | |
| # -c - see how much it apeared in each searched file | |
| # Grep certain file extensions | |
| grep -nir --include "*.js" --include "*.html" "searched term" ./ | |
| # Copy directory and replace links with real files | |
| cp -Lr some-dir some-new-dir | |
| # rsync - better then copy | |
| # Copy to the 10.0.50.50 server with specified credentials | |
| rsync -av -e "ssh -i /home/user/.ssh/servers-keys" /directory/which/should/be/copied 10.0.50.50:/directory/where/to/copy | |
| # Other Example. Resource: http://articles.slicehost.com/2007/10/9/backing-up-your-files-with-rsync | |
| rsync -e 'ssh -p 30000' -avl --delete --stats --progress demo@123.45.67.890:/home/demo /backup/ | |
| # Repeat command (ls -al) every 5 seconds | |
| watch -n 5 'ls -al' | |
| # curl send on OPTIONS request | |
| curl -I -v -H 'Host: www.example.com' -X OPTIONS http://www.example.com | |
| # View the command of the process matched with the id: PROCESS_ID | |
| ls -l /proc/{PROCESS_ID}/exe | |
| # Display all files except png's, gif's and jpg's | |
| # -f will show full paths | |
| tree -f -I '*gif|*jpg|*png' | |
| # Kill a process | |
| kill `ps -eo pid,args | awk '$3 == "myservice.java" { print $1 }'` | |
| # See which process is using the current directory | |
| fuser -v ./ | |
| # List process details | |
| ps -eo user,pid,%cpu,%mem,stat,start,time,cmd | |
| # list process relations | |
| ps auxw --forest | |
| # Sort by second column where the separator is a "," | |
| sort -t "," -k 2 input-file.data | |
| # Create a basic auth (htpasswd) file | |
| htpasswd -cdb file-name 'user-name' 'your-password' # File name is usually "htpasswd" | |
| # !!important!! -c will override the file if it exists. If You already have a file remove the -c option | |
| # monitor the system calls used by a program | |
| strace git ls-remote git@localhost:repo.git | |
| # -p flag to attach to a running process | |
| strace -p 1725 -o output.txt; tail -f output.txt | |
| # xargs example | |
| find . -name *.scss | xargs -I style_file echo "->" style_file | |
| # Strace save output to file | |
| strace -p 2707 -o /tmp/debug.log -f | |
| # View process memory details - memory map of a process | |
| pmap -x <PID> |
Integrate repo as subtree
git remote add -f remote-name git@server-name.com:repo-name.git
git subtree add --prefix local-directory-name remote-name branch-name --squash
Update
git fetch remote-name branch-name
git subtree pull --prefix local-directory-name remote-name branch-name --squash
Push
git subtree push --prefix=local-directory-name remote-name branch-name
Resources:
Get first tweet:
jq '.[0]' twitter-timeline.jsonGet tweets which belong to the user (are not "retweets"):
jq 'map(select(has("retweeted_status") == false)) | .[0]' twitter-timeline.jsonGet sorted tweets from 2016, which are not "retweets":
jq 'map(select(has("retweeted_status") == false)) | map({id: .id, created_at: .created_at, text: .text}) | sort_by(.id) | map(select(.created_at | contains("2016")))' twitter-timeline.json > twitter-timeline.2016.jsonFind content in a gzipped file
gzip -cd api.access_log-20130716.gz | grep -ni '16/Jul/2013:04' | less
Delete unwanted entries entries
sed '/crossdomain.xml/d' api.access_log > new-file.log
sed '/GET \/api\/version\/ HTTP\/1.1/d' api.access_log > new-file.log
Unzip file
gunzip api.access_log.gz
Add the following cron job:
*/30 * * * * /home/user_name/tmp/memory-usage-cron/bin/memory-usage.sh >/home/user_name/tmp/memory-usage-cron/log/cron.log 2>&1
The /home/user_name/tmp/memory-usage-cron/bin/memory-usage.sh file content:
#!/bin/bash
# Configure crontab
#
# */30 * * * * /home/user_name/tmp/memory-usage-cron/bin/memory-usage.sh >/home/user_name/tmp/memory-usage-cron/log/cron.log 2>&1
log_file_path='/home/user_name/tmp/memory-usage-cron/log'
ps ax -O rss | grep 'resque-1.25.1' | grep -v 'grep' | gawk '{ count ++; sum += $2 }; END {print "Total resque memory usage =", sum/1024, "MB";};' >> $log_file_path/resque.log
ps ax -O rss | grep 'mongo' | grep -v 'grep' | gawk '{ count ++; sum += $2 }; END {print "Total mongo memory usage =", sum/1024, "MB";};' >> $log_file_path/mongo.log
Show memory resident size - obszar, który siedzi w pamięci fizycznej, nie wirtualnej:
ps -o pid,rss,cmd ax | grep ruby
View what ocupies the memory on the address space level:
pmap -x <pid> | sort -k2 -r -n | head
Find all devices in the local network
nmap -sn 192.168.1.*
List open ports and processes that owns them
lsof -i
or
netstat -tulpn
Get the best of netstat
netstat -nltp
Get process which is bound to port 22
lsof -i :22
Show both listening and non-listening sockets
netstat -a
Show all TCP connections
netstat -t
Display all connections and don't resolve names
netstat -an
View processes which are using the network connection
netstat -p
Get listening ports
netstat -napt | grep -i LISTEN
Ask the DNS server where the domain can be found - ommit the cache
dig www.castaclip.de @ns1.syseleven.de
View the packets which go through port 18026 on localhost
tcpdump -s 0 -X port 18026 -i lo
-s - show the whole package
-X - print package data
-i lo - use the lo interface, ifconfig will show you the lo interface
View available interfaces with sudo tcpdump -D
Sniff on 10.0.50.2:9200
tcpdump -s 0 -X port 9200 and host 10.0.50.2 -i et0
Check for the et0 interface with ifconfig
Sniff and save to a file which later can be used with Wireshark
tcpdump -s 0 -w file-name.pcap -X port 9000 -i venet0
Find the file in: /var/lib/tcpdump/
Quick Wireshark Tips:
View only the answer section
dig redhat.com +noall +answer
View the MX DNS records
dig -t MX redhat.com +noall +answer
View only the NS DNS records
dig -t NS redhat.com +short
View all DNS records
dig -t ANY redhat.com +noall +answer
View only the IP of the A record
dig redhat.com +short
View the URL of the given IP (reverse lookup)
dig -x 209.132.183.81 +short
View TXT record
dig txt yandex-verification.domain.com
check the currently used tty
$ tty
/dev/pts/6
get the process id - PID
ps aux | grep 'you process name'
debbug Your process
$ gdb -p <PID>
attach Your current screen
(gdb) p close(1)
$1 = 0
(gdb) p fopen("/dev/pts/6", "w")
$1 = 0
(gdb) p close(2)
$1 = 0
(gdb) p fopen("/dev/pts/6", "w")
$1 = 0
(gdb) quit
Useful if You want to prewarm the cache.
Use the following command
wget -erobots=off -r -i url-list.txt
Example url-list.txt file
http://example.com
http://www.example.com
Download the java Linux x64 file archive from here.
Untar the archive and copy the directory into /opt/Oracle_Java/
Update alternatives:
# Set Java direcotry name - the name of the untarred archive
java_directory=jre1.8.0_25
sudo update-alternatives --install "/usr/bin/java" "java" "/opt/Oracle_Java/$java_directory/bin/java" 1
sudo update-alternatives --install "/usr/bin/javaws" "javaws" "/opt/Oracle_Java/$java_directory/bin/javaws" 1
sudo update-alternatives --set "java" /opt/Oracle_Java/$java_directory/bin/java
sudo update-alternatives --set "javaws" /opt/Oracle_Java/$java_directory/bin/javaws
sudo update-alternatives --install "/usr/lib/mozilla/plugins/mozilla-javaplugin.so" "mozilla-javaplugin.so" "/opt/Oracle_Java/$java_directory/lib/amd64/libnpjp2.so" 1
sudo update-alternatives --config mozilla-javaplugin.so
Check Java version by command line:
java -version
Check installed java version with Firefox here .
Resources:
Show git version:
git --version
Show global configuration variables:
git config --list
Configure git:
git config --global user.name "czerasz"
git config --global user.email "user@example.com"
git config --global color.ui true # Set code coloring
git config --global core.editor "vim"
Log about commits:
git log
git log --stat
Show commits and branches as a graph:
git log --graph --all
Nice way to show where you are currently:
git show
Add only changed files:
git add -u
Commit with an message:
git commit -m "some message"
View actual branch:
git branch
Show which branches were merged in actual branch:
git branch --merged
Delete branch:
git branch -d BRANCH_NAME
Create branch:
git branch BRANCH_NAME
Switch to another branch:
git checkout BRANCH_NAME
Create branch and switch to it:
git checkout -b BRANCH_NAME
Diff (compare) with the stash:
git diff stash@{0} master
git diff --name-only stash@{0} master
Show files in stash:
git show --name-only stash@{0}
Show a stashed file:
git show stash@{0}:<full filename relative to stash>
Show the latest commit hash:
git log --pretty=format:'%h' -n1
git log --pretty=format:'%H' -n1
Untrack and track file changes:
git update-index --assume-unchanged Gemfile.lock
git update-index --no-assume-unchanged Gemfile.lock
Move commits (from <commit 1> (included) to <commit 2>) to current branch:
git cherry-pick <commit 1>^..<commit 2>Debugging commands
strace -Ff -tt <program> <arguments> 2>&1 | tee strace.log
Or if the command is already running:
strace -Ff -tt -p `pidof <program>` 2>&1 | tee strace.log
For more details view this resource.
pkill -USR1 -n -x dd
dd if=~/Downloads/image.iso of=/dev/sdb
sync
Start a simple static server on port 8000:
python -m SimpleHTTPServer 8000
or
python3 -m http.server 8000
extension = "myfile.txt"[-3:]
a = [34, 'Fred', 12, False, 72.3]
for (i, x) in enumerate(a):
print(i, x)
l = ["abc", "def", "ghi", "ijk"]
[x.upper() for x in l]
>>> ['ABC', 'DEF', 'GHI', 'IJK']
x = 1.2345678
"x={:7.2f}".format(x)
>>> 'x= 1.23'
from datetime import datetime
d = datetime.now()
"{:%Y-%m-%d %H:%M:%S}".format(d)
>>> '2013-05-02 16:00:45'
def two_values():
return 1, 2
a, b = two_values()
class Person:
'''Documentation string'''
def __init__(self, name, tel):
self.name = name
self.tel = tel
Python 3:
class Employee(Person):
def __init__(self, first_name, surname, tel, salary):
super().__init__(first_name, surname, tel)
self.salary = salary
def give_raise(self, amount):
self.salary = self.salary + amount
Python 2:
class Employee(Person):
def __init__(self, first_name, surname, tel, salary):
Person.__init__(self, first_name, surname, tel)
self.salary = salary
def give_raise(self, amount):
self.salary = self.salary + amount
Todo
Find example which works in python 2 & 3
f = open('test.txt', 'w')
f.write('This file is not empty')
f.close()
| Mode | Description |
|---|---|
| r | Read |
| w | Write |
| a | Append |
| b | Binary mode |
| t | Text mode (default) |
| + | Shortcut for r+w |
The following example saves a complex list structure to file:
import pickle
mylist = ['some text', 123, [4, 5, True]]
f = open('list.pickle', 'w')
pickle.dump(mylist, f)
f.close()
To unpickle the contents of the file into a new list, use the following:
f = open('list.pickle')
imported_array = pickle.load(f)
f.close()
print(imported_array)
>>> ['some text', 123, [4, 5, True]]
try: f = open('test.txt')
s = f.read()
f.close()
except IOError:
print("Cannot open the file")
list = [1, 2, 3]
try:
list[8]
except:
print("out of range")
else:
print("in range")
finally:
print("always do this")
import sys
for (i, value) in enumerate(sys.argv):
print("arg: %d %s " % (i, value))
from time import sleep
import RPi.GPIO as GPIO
screen -S <SESSION NAME>
Note
To detach from a screen session use Ctrl+A + d
screen -ls
screen -dr <pid>
Where <pid> is the screen session PID.
Useful ~/.screenrc configuration:
defutf8 on
# Name the first tab
screen -t "First Tab"
# View the home directory in the "First Tab"
stuff "cd ~/ && clear^M"
# To use **Ctrl+B** instead of **Ctrl+A** add the following to the `~/.screenrc` file:
escape ^Bb
# Autodetach session on hangup instead of terminating screen completely
autodetach on
# Turn the splash screen off
startup_message off
# Use a 30000-line scrollback buffer
defscrollback 30000
# Set messages timeout to one second
msgwait 2
# Bind F11 and F12 (NOT F1 and F2) to previous and next screen window
bindkey -k F1 prev
bindkey -k F2 next
| #!/bin/bash | |
| script_directory="$( cd "$( dirname "$0" )" && pwd )" | |
| project_directory=$script_directory/.. |
Check if the system has any configured swap:
swapon -s
Check available disc space:
df -h
Create a Swap File :
fallocate -l 1G /swapfile
chmod 600 /swapfile
Set up the swap space:
mkswap /swapfile
Enable the swap space:
swapon /swapfile
Verify:
swapon -s
Make the Swap File Permanent:
echo -e "\n\n/swapfile none swap sw 0 0" >> /etc/fstab
Fine tune:
sysctl vm.swappiness=10
echo "vm.swappiness=10" | tee /etc/sysctl.conf -a
sysctl vm.vfs_cache_pressure=50
echo "vm.vfs_cache_pressure=50" | tee /etc/sysctl.conf -a
Resources:
Display content from line number starting_line_number to line number ending_line_number
sed -n {starting_line_number},{ending_line_number}p filename.log
View the lines from 10 to 20
sed -n 10,20p /logs-path/file.log
Append text to a line containing string
sed -i -e '/searched text/s/$/ Text which You want append/' input-file.data
Prepend text to a line containing string
sed -i -e '/searched text/s/^/Text which You want prepend /' input-file.data
sed -e 'p' -e 'd' input-file.data
Is the same as:
sed 'p; d' input-file.data
d deletes the line from the pattern space.
After d all other commands are ignored
sed -n 's/change text/to this text/' input-file.data
-n - supress the output - don't view the changes
Find replace and overwrite file
sed -i 's/change text/to this text/' file-name.data
Find replace from line 3 to 10
sed '3,10s/change text/to this text/' input-file.data
Substitute only lines which contain "change text" - p flag
sed -n 's/change text/to this text/p' input-file.data
Ignore case - I flag
sed 's/CHANGE TEXT/to this text/I' input-file.data
Change letters - change s to S and k to K
sed 'y/sk/SK/' input-file.data
Get the script from the script.sed file
sed -f script.sed input-file.data
script.sed content:
# In the sed script file You can use comments
s/some text/other text/Iw file-name.out
zcat file.gz | head -250
Display lines matching a pattern, and few lines following the match
grep -A 5 "searched value" filename.log
Options:
N lines after the match with - A optionN lines before the match with - B optionN lines around the match with - C optionsed -n -e 10p -e 21p filename.log
sed -n -e 10p filename.log
gzip example:
gzip -c dist/css/screen.css > dist/css/screen.css.gz
Backup file example:
cp init.js init.tmp.`date +"%y%m%d%H%M"`.js
Loop example
sites=(site-name-1 site-name-2 site-name-3)
for site in "${sites[@]}"
do
if [ -d "$site/css/version-1" ]; then
echo "css already exists on $site"
else
echo "css don't exists on $site"
cp -r global/css/version-1/ $site/css
fi
done
Application benchmark:
ab -k -c 10 -t 60 -n 100000000 -s 15 \
-H 'Host: example.com' \
http://localhost/
| Option | Parameter | Description |
|---|---|---|
| Concurrency | -c |
10 parallel users |
| Time for Test | -t |
60 seconds |
| Timeout | -s |
15 seconds (30 seconds it kind of too high) - this value is the timeout per request |
| Number of Requests | -n |
100 million (All will not be processed in the 60s - therefore this is a good upper limit, to ensure that the test is run for 60s) |
| Header | -H |
HTTP request header |
Note: Always use
-tBEFORE-n
ssh -f user@first-server.com -L 5500:second-server.com:22 -N # 5500 - local port
scp -r -P5500 /path/to/file user-for-the-second-server@localhost:/path/where/to/copy
ssh -i ~/.ssh/key_rsa user@first-server.com -L15432:second-server.com:5432 -L25432:third-server.com:5432
sshfs user@server.uri.com: /home/where/to/mount/on/host/ -o "IdentityFile=~/.ssh/key_rsa"
/usr/sbin/sshd -p 2022 -D
Note: !! Do this always when You play with ssh configuration !!
To restrict user only to a specific command add his key to ~/.ssh/authorized_keys with the following prefix:
command="/usr/bin/ssh ${SSH_ORIGINAL_COMMAND#* }" ssh-rsa abcdef...
Where abcdef... is the ssh key and /usr/bin/ssh is the restricted command
Add to the end of pg_hba.conf (find it with sudo find / -name pg_hba.conf -type f) on vagrant:
# TYPE DATABASE USER CIDR-ADDRESS METHOD
# "local" is for Unix domain socket connections only
local all all trust
# IPv4 local connections:
host all all 127.0.0.1/32 trust
# IPv6 local connections:
host all all ::1/128 trust
host all all 33.33.33.20/32 trust
Then create the tunnel on Your host:
ssh -f -N vagrant@33.33.33.20 -L5555:localhost:5432
Messages are going to 10.0.79.9:L5556 via a tunnel over proxy.server.net.
The message is comming back over proxy.server.net:55562 (10.0.79.1:55562) to the local machine on port 55562.
ssh user@proxy.server.net -L5556:10.0.79.9:5556 -R10.0.79.1:55562:127.0.0.1:55562
Open a tunnel from the local mashine (port 3022) to the domain.net server. You should now be able to connect through the domain.net server to the machine. Everything what comes to domain.net:81 will be redirected to the local 3022 port.
ssh autossh@domain.net -R 3022:localhost:22
autossh is the user with a shell /etc/bin/autossh