Skip to content

Instantly share code, notes, and snippets.

@danielbraun
Last active April 17, 2020 16:08
Show Gist options
  • Save danielbraun/e1534ce5da56894c4fce15a16016059b to your computer and use it in GitHub Desktop.
Save danielbraun/e1534ce5da56894c4fce15a16016059b to your computer and use it in GitHub Desktop.
My magic makefile to make everything happen
SPACY_MODEL ?= xx_ent_wiki_sm
PYTHONPATH ?= ~/django_home
DJANGO_SETTINGS_MODULE ?= django_home.settings
%.min.js: %.js
uglifyjs < $< > $@
%.xml: %.edn
edn2xml < $< > $@
%.bundle.js: %.js
browserify $< > $@
%.tab: %.mysql.sql
mysql --defaults-file=my.cnf < $< > $@
%.xml: %.mysql.sql
mysql --defaults-file=my.cnf --xml < $< > $@
%.html: %.psql.sql
psql --html < $< > $@
%.xml: %.psql.sql
# TODO fix double quotes
echo "select concat('<?xml version=\"1.0\"?>',query_to_xml(:q, false, false, ''));" \
| psql -t -A -v q="'$(shell cat $<)'" \
> $@
%.xml: %.html
xsltproc --html $*.xsl $< -o $@
%.txt: %.pdf
pdftotext - - < $< > $@
%.info.txt: %.pdf
pdfinfo $< > $@
%.pdf: %.svg
cairosvg $< -o $@
%.out: %.curl
curl -K $< > $@
%.models.py:
PYTHONPATH=$(PYTHONPATH) DJANGO_SETTINGS_MODULE=$(DJANGO_SETTINGS_MODULE) python3 -m django inspectdb $(shell basename $*) > $@
U.S.DividendChampions.xlsx:
curl -L https://bitly.com/USDividendChampions > $@
%.dividends.csv:
curl "https://query1.finance.yahoo.com/v7/finance/download/$(shell basename $*)?period1=0&period2=$(shell date +%s)&interval=1d&events=div" > $@
%.sorted.csv: %.csv
csvsort < $< > $@
%/stocks.nyse.min.json:
curl 'https://www.nyse.com/api/quotes/filter' \
-XPOST \
-H 'Content-Type: application/json' \
--data-binary '{"instrumentType":"EQUITY","pageNumber":1,"maxResultsPerPage":100000}' > $@
%/dividends: %/stock_symbols.nyse.csv.tail
mkdir -p $@
cat $< | xargs printf "$@/%s.dividends.sorted.csv\n" | xargs $(MAKE)
%/stock_symbols.nyse.csv: %/stocks.nyse.csv
cat $< | csvcut -c symbolTicker > $@
%.tail: %
tail -n +2 < $< > $@
%.csv: %.json
json2csv < $< > $@
%.json: %.iob
python3 -m spacy convert $< > $@
%.table.txt: %.csv
csvlook $@ > $@
%.json: %.min.json
jq < $< > $@
%.min.json: %.csv
csvjson < $< > $@
%.out.edn: %.edn
%.xml: %.php
php < $< > $@
%.svg: %.csv
gnuplot -e "set key autotitle columnhead; set datafile separator ','; set terminal svg enhanced mouse jsdir 'http://gnuplot.sourceforge.net/demo_svg_5.2'; plot '$<';" > $@
%.instagram.jpg:
curl -L "https://www.instagram.com/p/$(shell basename $*)/media?size=l" > $@
# Sign in doesn't work
music/%:
@mkdir -p `dirname $@`
printf "<iframe \
src='https://embed.music.apple.com/us/album/$(shell basename $*)' \
height='100%%' \
frameborder='0' \
allow='autoplay *; encrypted-media *;' \
sandbox='allow-forms allow-popups allow-same-origin allow-scripts allow-storage-access-by-user-activation allow-top-navigation-by-user-activation' \
width='100%%' \
></iframe>" > $@
youtube/%:
@mkdir -p `dirname $@`
printf "<iframe \
src='https://www.youtube.com/embed/$(shell basename $*)' \
height='100%%' \
frameborder='0' \
width='100%%' \
></iframe>" > $@
yad2/rentals/%.json:
@mkdir -p `dirname $@`
@res="$(shell curl 'https://www.yad2.co.il/api/pre-load/getFeedIndex/realestate/rent?page=$*' \
-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/$*.1.15 (KHTML, like Gecko) Version/12.1.2 Safari/605.1.15' \
-f \
-L \
-s \
-w %{url_effective} \
-o $@)" \
; file --mime $@ | grep "text/html" \
&& echo "Error: Verification required" \
&& rm $@ && open $$res\
&& exit 1 \
|| exit 0
domains/%:
mkdir -p $(shell dirname $@)
whois $* > $@
nslookup $* >> $@
%.svg: %.geojson
geo2svg < $< > $@
%.kml: %.geojson
./node_modules/.bin/tokml $< > $@
server:
shell2http -cgi / 'export FILE=.`echo $$REQUEST_URI | sed "s/?.*//"` && make -s $$FILE && cat $$FILE'
corona.geojson:
curl "https://services5.arcgis.com/dlrDjz89gx9qyfev/arcgis/rest/services/Corona_Exposure_View/FeatureServer/0/query?where=1%3D1&objectIds=&time=&geometry=&geometryType=esriGeometryEnvelope&inSR=&spatialRel=esriSpatialRelIntersects&resultType=none&distance=0.0&units=esriSRUnit_Meter&returnGeodetic=false&outFields=*&returnGeometry=true&featureEncoding=esriDefault&multipatchOption=xyFootprint&maxAllowableOffset=&geometryPrecision=&outSR=&datumTransformation=&applyVCSProjection=false&returnIdsOnly=false&returnUniqueIdsOnly=false&returnCountOnly=false&returnExtentOnly=false&returnQueryGeometry=false&returnDistinctValues=false&cacheHint=false&orderByFields=&groupByFieldsForStatistics=&outStatistics=&having=&resultOffset=&resultRecordCount=&returnZ=false&returnM=false&returnExceededLimitFeatures=true&quantizationParameters=&sqlFormat=none&f=pgeojson&token=" -o $@
%.leaflet.json: %.geojson
jq -s ".[0].geoJSON=.[1] | first" leaflet.json $< > $@
%.json: %.xml
xml2json < $< > $@
%.json.js: %.json
browserify --require ./$< -s $(shell basename $*) > $@
%.json.xml: %.json
echo "<?xml version=\"1.0\" encoding=\"UTF-8\"?><?xml-stylesheet href=\"leaflet.xsl\" type=\"text/xsl\"?><root><![CDATA[$$(cat $<)]]></root>" > $@
%.leaflet.html: $.leaflet.json
%.sql: %.csv
csvsql $< > $@
airbnb/%:
curl "https://www.airbnb.com/api/v2/pdp_listing_booking_details?_format=for_web_with_date&check_in=2020-03-29&check_out=2020-04-21&currency=ILS&guests=1&key=d306zoyjsyarp7ifhu67rjxn52tv0t20&listing_id=$*&locale=en&number_of_adults=1&number_of_children=0&number_of_infants=0" \
| jq ".pdp_listing_booking_details[0].price.total.amount" \
> $@
%.spacy.json: %.txt
python3 -c'import spacy, sys, json; print(json.dumps(spacy.load("$(SPACY_MODEL)")("".join(sys.stdin.readlines())).to_json()))' < $< | jq > $@
%.ent.spacy.html: %.txt
python3 -c 'import spacy, sys; print(spacy.displacy.render([spacy.load("$(SPACY_MODEL)")("".join(sys.stdin.readlines()))], style="ent", page=True))' < $< > $@
%.png: %.pdf
cd $(shell dirname $@) && pdftocairo -png -singlefile $(shell basename $<)
dividend-aristocrats.csv:
curl -L http://www.simplysafedividends.com/intelligent-income/idea_lists/3-dividend-aristocrats.csv > $@
%.test.iob: %.iob
cat $< | head -n $(shell echo $(shell cat $< | wc -l) \* 0.6 / 1 | bc) > $@
%.dev.iob: %.iob
cat $< | head -n $(shell echo $(shell cat $< | wc -l) \* 0.4 / 1 | bc) > $@
%.model: %.test.json %.dev.json
rm -rf $@
python3 -m spacy train \
-p ner \
-b xx_ent_wiki_sm\
xx $@ $*.test.json $*.dev.json
clean:
git clean -fdX
.PHONY: %/dividends *.php clean
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment