`just create_and_start` to bring up a quick network

README.md

Create a network with 250 apps and 1000 boxes - HOWTO

(Or with BOXES_PER_APP set to 2048 as it currently is, this will produce thousands of boxes, but take a very long time)

• just is a make copycat that is more suitable for scripting as opposed to building stuff
• you can install just by following the pre-built binary installation instructions
• the code is avaiable at casey's repo
• this example shows how I bootstrapped a network with 250 apps with a total of 1000 boxes for the purposes of testing ingestion by indexer. To replicate what I did:
  • see if you have a directory called networks in your home directory, or don't mind the script creating one for you (otherwise change the NETWORKS variable in the script)
  • don't have a network called indexerboxes, as the script will attempt to create it (otherwise change the NAME variable to a name that doesn't exist in ~/networks -assuming you didn't change the value of NETWORKS in the previous step)
  • run just create_and_start. This creates the network ~/$NETWORKS/$NAME and starts running it
  • run just gen-mult-app-boxes 250. This creates 250 apps along with around 250 X 4 boxes associated to those apps. You can supply a different number of apps and not supplying any will default to 10 apps with 25 boxes.
  • when that script finishes (could take an hour!) you should run just stop to stop running the network. If you want to delete it entirely, you can run just nuke.
• to see all available just commands, simple run just without any arguments

boxes.teal

// WARNING: THIS IS NOT A PRODUCTION QUALITY PROGRAM - FOR TEST PURPOSES ONLY

#pragma version 7
    txn ApplicationID
    bz end
    txn ApplicationArgs 0   // [arg[0]] // fails if no args && app already exists
    byte "create"			// [arg[0], "create"] // create box named arg[1]
    ==                      // [arg[0]=?="create"]
    bz del                  // "create" ? continue : goto del
    int 24                  // [24]
    txn NumAppArgs          // [24, NumAppArgs]
    int 2                   // [24, NumAppArgs, 2]
    ==                      // [24, NumAppArgs=?=2]
    bnz default             // WARNING: Assumes that when "create" provided, NumAppArgs == 2
    pop						// get rid of 24 // NumAppArgs != 2
    txn ApplicationArgs 2   // [arg[2]]
    btoi                    // [btoi(arg[2])]
default:                    // [24] // NumAppArgs == 2
    txn ApplicationArgs 1   // [24, arg[1]]
    box_create              // [] // boxes: arg[1] -> [24]byte
    b end
del:						// delete box arg[1]
    txn ApplicationArgs 0   // [arg[0]]
    byte "delete"           // [arg[0], "delete"]
    ==                      // [arg[0]=?="delete"]
    bz set                  // "delete" ? continue : goto set
    txn ApplicationArgs 1   // [arg[1]]
    box_del                 // del boxes[arg[1]]
    b end
set:						// put arg[1] at start of box arg[0] ... so actually a _partial_ "set"
    txn ApplicationArgs 0   // [arg[0]]
    byte "set"              // [arg[0], "set"]
    ==                      // [arg[0]=?="set"]
    bz test                 // "set" ? continue : goto test
    txn ApplicationArgs 1   // [arg[1]]
    int 0                   // [arg[1], 0]
    txn ApplicationArgs 2   // [arg[1], 0, arg[2]]
    box_replace             // [] // boxes: arg[1] -> replace(boxes[arg[1]], 0, arg[2])
    b end
test:						// fail unless arg[2] is the prefix of box arg[1]
    txn ApplicationArgs 0   // [arg[0]]
    byte "check"            // [arg[0], "check"]
    ==                      // [arg[0]=?="check"]
    bz bad                  // "check" ? continue : goto bad
    txn ApplicationArgs 1   // [arg[1]]
    int 0                   // [arg[1], 0]
    txn ApplicationArgs 2   // [arg[1], 0, arg[2]]
    len                     // [arg[1], 0, len(arg[2])]
    box_extract             // [ boxes[arg[1]][0:len(arg[2])] ]
    txn ApplicationArgs 2   // [ boxes[arg[1]][0:len(arg[2])], arg[2] ]
    ==                      // [ boxes[arg[1]][0:len(arg[2])]=?=arg[2] ]
    assert                  // boxes[arg[1]].startwith(arg[2]) ? pop : ERROR
    b end
bad:
    err
end:
    int 1

clear.teal

#pragma version 7
int 1

Justfile

set export
set shell := ["zsh", "-cu"]

NETWORKS := `echo $HOME` + "/networks"
# NAME := "indexerboxes"
NAME := "niftynetwork"
TEMP_NETWORK := NETWORKS + "/" + NAME
# GO_ALGORAND := "../../.."
GO_ALGORAND := ".."
NODE_TEMPLATE := GO_ALGORAND + "/test/testdata/nettemplates/OneNodeFuture.json"
ALGORAND_DATA := TEMP_NETWORK + "/Primary"
BOXES_TEAL := "boxes.teal"

# --- SUMMARY --- #

# list all available commands
default:
    just --list

# echo all variables
@echo:
    echo NETWORKS: $NETWORKS
    echo NAME: $NAME
    echo TEMP_NETWORK: $TEMP_NETWORK
    echo GO_ALGORAND: $GO_ALGORAND
    echo NODE_TEMPLATE: $NODE_TEMPLATE
    echo ALGORAND_DATA: $ALGORAND_DATA
    echo BOXES_TEAL: $BOXES_TEAL

# --- GENERATOR SCRIPT COMMANDS --- #

# generate an arbitrary number of app and box scenarios, each with up to BOXES_PER_APP boxes
gen-mult-app-boxes NUM_APPS="10" BOXES_PER_APP="2048":
    #!/usr/bin/env python3
    import subprocess
    from subprocess import CalledProcessError

    num_apps = int({{NUM_APPS}})
    print(f"{num_apps=}")
    for i in range(num_apps):
        print("\n", "\n", "\n", f"gen-app-and-box-scenarios #{i+1}" )
        subprocess.run(["just", "gen-app-and-box-scenarios", "{{BOXES_PER_APP}}"]).check_returncode()


# create an app and add up to BOXES_PER_APP random boxes to it in a multi-threaded fashion
gen-app-and-box-scenarios BOXES_PER_APP="10":
    #!/usr/bin/env python3
    from concurrent.futures import ThreadPoolExecutor
    import json
    import logging
    import random
    import string
    import subprocess
    from subprocess import CalledProcessError
    import time

    CHARS = string.digits + string.ascii_letters
    VAL_SIZE = 24
    BOXES_PER_APP = int({{BOXES_PER_APP}})
    NLS = "\n" * 3

    subprocess.run(["just", "app-create_fund"]).check_returncode()

    def worker(thread_number):
        logging.info(f"HELLO from {thread_number}!")
        
        create_cpe = set_cpe = test_cpe = del_cpe = None

        rand_key_size = random.randint(4, 64)
        rand_key = "".join(random.choice(CHARS) for _ in range(rand_key_size))
        print(f"{NLS}{thread_number}: {rand_key=}")
        try:
            subprocess.run(["just", "box-create", rand_key]).check_returncode()
        except CalledProcessError as cpe:
            create_cpe = str(cpe)

        rand_val = "".join(random.choice(CHARS) for _ in range(VAL_SIZE))
        print(f"{NLS}{thread_number}: {rand_val=}")
        try:
            subprocess.run(["just", "box-set", rand_key, rand_val]).check_returncode()
        except CalledProcessError as cpe:
            set_cpe = str(cpe)

        print(f"{NLS}{thread_number}: checking {rand_val=}")
        try:
            subprocess.run(["just", "box-test", rand_key, rand_val]).check_returncode()
        except CalledProcessError as cpe:
            test_cpe = str(cpe)

        delete = random.choice([True, False])
        if delete:
            print(f"{NLS}{thread_number}: deleting")
            try:
                subprocess.run(["just", "box-delete", rand_key]).check_returncode()
            except CalledProcessError as cpe:
                del_cpe = str(cpe)

        return {
            "thread_number": thread_number,
            "key_size": rand_key_size,
            "key": rand_key,
            "val": rand_val,
            "deleted": delete,
            "called_process_errors": {
                "create_cpe": create_cpe,
                "set_cpe": set_cpe,
                "test_cpe": test_cpe,
                "del_cpe": del_cpe,
            },
        }

    format = "%(asctime)s: %(message)s"
    logging.basicConfig(format=format, level=logging.INFO,
                        datefmt="%H:%M:%S")

    results = []
    with ThreadPoolExecutor() as executor:
        for r in executor.map(worker, range(BOXES_PER_APP)):
            results.append(r)

    print(json.dumps(results, indent=2))
    
    for result in results:
        for err in result["called_process_errors"].values():
            if err:
                raise err

# --- HIGHER LEVEL --- #

# create and then start (error if already created)
@create_and_start: create start status
    sleep 5
    just status

# create an app and then fund it
@app-create_fund: app-create last-app-fund 


# --- BOX PUT: HIGHER LEVEL --- #

# create box[BOX] for last app with provided key variable BOX
@box-create $BOX:
    just app-call-last '\"create\", \"{{BOX}}\"'

# set box[BOX]=VAL for last app with key BOX and val VAL
@box-set $BOX $VAL:
    just app-call-last '\"set\", \"{{BOX}}\", \"{{VAL}}\"'

# set box[BOX]=VAL for last app with key BOX and val VAL
@box-test $BOX $VAL:
    just app-call-last '\"check\", \"{{BOX}}\", \"{{VAL}}\"'

# delete box[BOX] for last app
@box-delete $BOX:
    just app-call-last '\"delete\", \"{{BOX}}\"'

# stop and tear down the node network. WARNING: YOU WILL LOSE ALL YOUR NODE DATA FROM THE FILE SYSTEM.
@stop_and_nuke: stop nuke

# --- PRE-REQUISITES --- #

# calculate an app's address using the python SDK
app-address *ARGS:
    #!/usr/bin/env python3
    from algosdk import logic
    print(logic.get_application_address({{ ARGS }}))

# --- NETWORKS / NODE --- #

# create a private network with one node (error if already created)
@create:
	mkdir -p $NETWORKS
	goal network create -n $NAME -r $TEMP_NETWORK -t $NODE_TEMPLATE

# start a the network (error if already running or not created)
@start:
	goal node start

# status of network node
@status:
	goal node status && echo "RUNNING" || echo "NOT RUNNING"

# stop the running node (error if not running)
@stop:
	goal node stop

# remove the node's data from the file system
@nuke:
    echo "deleting $TEMP_NETWORK"
    rm -rf $TEMP_NETWORK

# --- ACCOUNTS --- #

# list all associated accounts
@list:
	goal account list

# create a new account without renaming it to a human friendly local alias
@raw-new-account:
    goal account new | awk '{print $NF}'

@account-alias $ACCOUNT:
    just list | grep {{ACCOUNT}} | awk '{print $2}'

# create a new locally aliased account
@new-account $ALIAS $ACCOUNT=`just raw-new-account`:
    goal account rename `just account-alias {{ACCOUNT}}` {{ALIAS}}

# create a new multisig account with threshold 1 using provided accounts (cannot handle aliases)
@raw-msig-account *ACCOUNTS:
    goal account multisig new -T 1 {{ACCOUNTS}} | awk '{print $NF}'

# create a new multisig account with given ALIAS and threshold 1 using provided accounts (cannot handle aliases)
@new-msig-account $ALIAS *ACCOUNTS:
    goal account rename `just account-alias $(just raw-msig-account {{ACCOUNTS}})` {{ALIAS}}
 
# funding account's address
@funder:
    just list | awk '{print $2}'
    
# provide information about a given account
@info $ACCOUNT=`just funder`:
    goal account info --address {{ACCOUNT}}

# provide an account's balance
@balance $ACCOUNT=`just funder`:
    goal account balance --address {{ACCOUNT}}

# funder's most recently created app-id
@last-app-id:
    just info | grep ID | tail -n 1 | cut -d "," -f1 | awk '{print $2}'

# the account address of the funders most recently created app-id
@last-app-address:
    just app-address `just last-app-id`

# --- ASSETS --- #

# create a dummy asset for the provided FUNDER. Copy pasta from: https://dappradar.com/blog/algorand-dapp-development-2-standard-asset-management
@asset-create $FUNDER=`just funder`:
    goal asset create --creator {{FUNDER}} --total 1000000 --unitname bUSD --name "Balgorand USD" --asseturl "https://b-usd.com" --decimals 9

# --- APPLICATIONS --- #

# information about an application of given id
@app-info $APP_ID=`just last-app-id`:
    goal app info --app-id {{APP_ID}}

# print out the boxes teal program
@boxes_teal:
    cat $BOXES_TEAL

# shortcut for the approval and clear program `goal app create` params
@programs:
    echo "--approval-prog $BOXES_TEAL --clear-prog clear.teal"

# shortcut for the storage params of `goal app create`
@app-vars $GBS="0" $GI="0" $LBS="0" $LI="0":
    echo "--global-byteslices $GBS --global-ints $GI --local-byteslices $LBS --local-ints $LI"

# shortcut for creating the arguments of an app call
box-app-args *ARGS:
    #!/usr/bin/env python3
    args = []
    box_arg = ""
    i = 0
    for arg in {{ARGS}}:
        try:
            int(arg)
            arg = f"int:{arg}"
        except Exception:
            arg = f"str:{arg}"
        args.extend(["--app-arg", arg])
        if i == 1:
            box_arg = f"--box {arg}"
        i += 1
    if box_arg:
        args = [box_arg] + args
    print(*args)

# create an app funded by funder account
@app-create $GBS="0" $GI="0" $LBS="0" $LI="0":
    echo "goal app create --creator `just funder` `just programs` `just app-vars $GBS $GI $LBS $LI`"
    goal app create --creator `just funder` `just programs` `just app-vars $GBS $GI $LBS $LI`

# call the last app from the funder address using ARGS
@app-call-last *ARGS='\"create\", \"mybox\"':
    (set -x; goal app call --app-id `just last-app-id` --from `just funder` `just box-app-args {{ARGS}}`)

# --- BOX INFO --- #

# get all the boxes associated a given app-id
app-box-list $APP_ID=`just last-app-id`:
    goal app box list --app-id {{APP_ID}} --max 0

# get box information for agiven app-id and box anme
app-box-info $APP_ID=`just last-app-id` $BOX="str:mybox":
    goal app box info --app-id {{APP_ID}} --name {{BOX}}


# --- CLERK --- #

# send from one account to another a given amount
@send $FROM $TO $AMOUNT:
    goal clerk send --from {{FROM}} --to {{TO}} --amount {{AMOUNT}}

# fund the most recently created app
@last-app-fund $AMOUNT=`echo 133713371337`:
    just send `just funder` `just last-app-address` {{AMOUNT}}

# --- CONSENSUS PARAMS --- #

# list all consensus param declarations
@consensus-params-list:
    cat ${GO_ALGORAND}/config/consensus.go | egrep " (bool|byte|int$|uint|map\[|Duration|PaysetCommitType)" | egrep -v "type|=|func|,|//"

# print out the value history of a consensus param
@consensus-param $CP="MaximumMinimumBalance":
    cat ${GO_ALGORAND}/config/consensus.go | grep {{CP}} || echo "{{CP}} not found in consensus.go"

# consensus params for program size
consensus-prog-size:
    just consensus-param LogicSigMaxSize
    just consensus-param MaxAppProgramLen

# consensus param for LogicSigMaxCost
consensus-prog-cost:
    just consensus-param MaxCost
    just consensus-param MaxAppProgramCost

# consensus params for program pages
@consensus-prog-pages:
    just consensus-param MaxExtraAppProgramPages

# consnsus params for foreign refs:
consensus-foreign-refs:
    just consensus-param MaxAppTxnAccounts
    just consensus-param MaxAppTxnForeignApps
    just consensus-param MaxAppTxnForeignAssets
    just consensus-param MaxAppTotalTxnReferences
    just consensus-param MaxAppBoxReferences

# consensus params for local/global storage:
consensus-storage:
    just consensus-param MaxAppKeyLen
    just consensus-param MaxAppBytesValueLen
    just consensus-param MaxAppSumKeyValueLens
    just consensus-param MaxLocalSchemaEntries
    just consensus-param MaxGlobalSchemaEntries

# consensus params for min-balance calc:
consensus-minbal:
    just consensus-param SchemaMinBalancePerEntry
    just consensus-param SchemaUintMinBalance
    just consensus-param SchemaBytesMinBalance
    just consensus-param BoxFlatMinBalance
    just consensus-param BoxByteMinBalance

# consensus params for boxes:
consensus-boxes:
    just consensus-param MaxAppKeyLen
    just consensus-param MaxBoxSize
    just consensus-param BoxFlatMinBalance
    just consensus-param BoxByteMinBalance
    just consensus-param MaxAppBoxReferences
    just consensus-param BytesPerBoxReference

# --- MISCELLANEOUS --- #

# print out the network's algod & kmd token and network/process info
@client-info:
    echo "algod.token: "$(cat ${ALGORAND_DATA}/algod.token)
    echo "algod.net:   "$(cat ${ALGORAND_DATA}/algod.net)
    echo "algod.pid:   "$(cat ${ALGORAND_DATA}/algod.pid)
    echo "kmd.token:   "$(cat ${ALGORAND_DATA}/kmd-v0.5/kmd.token)
    echo "kmd.net---->"
    cat ${ALGORAND_DATA}/kmd-v0.5/kmd.log | grep 127.0.0.1 | head -n 1 | cut -d '"' -f4

# print out broadcastQueueBulk's channel size ... the default is 100 which is too small for the example
@broadcast-queue-size:
    echo "default is 100. What is it actually in network/wsNetwork.go ?"
    cat {{GO_ALGORAND}}/network/wsNetwork.go | grep "wn.broadcastQueueBulk = make(chan broadcastRequest" | cut -d "," -f2 | cut -d ")" -f1 | awk '{print $1}'

Makefile.INDEXER

#### THIS IS WHAT I'm USING IN INDEXER TO INTERACT WITH THE ACCOMPANYING justfile GOAL SCRIPT ####

SRCPATH		:= $(shell pwd)
VERSION		:= $(shell $(SRCPATH)/mule/scripts/compute_build_number.sh)
OS_TYPE		?= $(shell $(SRCPATH)/mule/scripts/ostype.sh)
ARCH		?= $(shell $(SRCPATH)/mule/scripts/archtype.sh)
PKG_DIR		= $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH)/$(VERSION)
ifeq ($(OS_TYPE), darwin)
ifeq ($(ARCH), arm64)
export CPATH=/opt/homebrew/include
export LIBRARY_PATH=/opt/homebrew/lib
endif
endif

# TODO: ensure any additions here are mirrored in misc/release.py
GOLDFLAGS += -X github.com/algorand/indexer/version.Hash=$(shell git log -n 1 --pretty="%H")
GOLDFLAGS += -X github.com/algorand/indexer/version.Dirty=$(if $(filter $(strip $(shell git status --porcelain|wc -c)), "0"),,true)
GOLDFLAGS += -X github.com/algorand/indexer/version.CompileTime=$(shell date -u +%Y-%m-%dT%H:%M:%S%z)
GOLDFLAGS += -X github.com/algorand/indexer/version.GitDecorateBase64=$(shell git log -n 1 --pretty="%D"|base64|tr -d ' \n')
GOLDFLAGS += -X github.com/algorand/indexer/version.ReleaseVersion=$(shell cat .version)

COVERPKG := $(shell go list ./...  | grep -v '/cmd/' | egrep -v '(testing|test|mocks)$$' |  paste -s -d, - )

# Used for e2e test
export GO_IMAGE = golang:$(shell go version | cut -d ' ' -f 3 | tail -c +3 )

# This is the default target, build the indexer:
cmd/algorand-indexer/algorand-indexer: idb/postgres/internal/schema/setup_postgres_sql.go go-algorand
	cd cmd/algorand-indexer && go build -ldflags="${GOLDFLAGS}"

# DO NOT MERGE THIS!
go-algorand: go-avm-box
	cd third_party/go-algorand && \
		make crypto/libs/`scripts/ostype.sh`/`scripts/archtype.sh`/lib/libsodium.a

idb/postgres/internal/schema/setup_postgres_sql.go:	idb/postgres/internal/schema/setup_postgres.sql
	cd idb/postgres/internal/schema && go generate

idb/mocks/IndexerDb.go:	idb/idb.go
	go install github.com/vektra/mockery/v2@v2.12.3
	cd idb && mockery --name=IndexerDb

# check that all packages (except tests) compile
check: go-algorand
	go build ./...

package: go-algorand
	rm -rf $(PKG_DIR)
	mkdir -p $(PKG_DIR)
	misc/release.py --host-only --outdir $(PKG_DIR)

# used in travis test builds; doesn't verify that tag and .version match
fakepackage: go-algorand
	rm -rf $(PKG_DIR)
	mkdir -p $(PKG_DIR)
	misc/release.py --host-only --outdir $(PKG_DIR) --fake-release

test: idb/mocks/IndexerDb.go cmd/algorand-indexer/algorand-indexer
	go test -coverpkg=$(COVERPKG) ./... -coverprofile=coverage.txt -covermode=atomic ${TEST_FLAG}

lint: go-algorand
	golint -set_exit_status ./...
	go vet ./...

fmt:
	go fmt ./...

integration: cmd/algorand-indexer/algorand-indexer
	mkdir -p test/blockdata
	curl -s https://algorand-testdata.s3.amazonaws.com/indexer/test_blockdata/create_destroy.tar.bz2 -o test/blockdata/create_destroy.tar.bz2
	test/postgres_integration_test.sh

e2e: cmd/algorand-indexer/algorand-indexer
	cd misc && docker-compose build --build-arg GO_IMAGE=${GO_IMAGE} && docker-compose up --exit-code-from e2e

deploy:
	mule/deploy.sh

sign:
	mule/sign.sh

test-package:
	mule/e2e.sh

test-generate:
	test/test_generate.py

nightly-setup:
	cd third_party/go-algorand && git fetch && git reset --hard origin/master

nightly-teardown:
	git submodule update

indexer-v-algod-swagger:
	pytest -sv misc/parity

indexer-v-algod: nightly-setup indexer-v-algod-swagger nightly-teardown

.PHONY: test e2e integration fmt lint deploy sign test-package package fakepackage cmd/algorand-indexer/algorand-indexer idb/mocks/IndexerDb.go go-algorand indexer-v-algod

# TEMPORARY RECIPE. DO NOT MERGE.
go-avm-box:
	echo "HELLO, this is a NO-OP currently"
	# test -h third_party/go-algorand && exit 0 || exit 1


LOCAL_GO_ALGORAND = "/Users/zeph/github/tzaffi/go-algorand"
# "/Users/zeph/github/cce/go-algorand"
zymlocal:
	mv third_party/go-algorand third_party/zo-algorand
	ln -s $(LOCAL_GO_ALGORAND) third_party/go-algorand

dezymlocal:
	rm third_party/go-algorand
	mv third_party/zo-algorand third_party/go-algorand

ALGOD_TOKEN = eb7def8f9a3c34a2c982fc211d21ad2ce042ec6809b8b283d1b9c233ae8ed441
INDEXER_DB = "host=127.0.0.1 user=postgres password=zephr0cks dbname=indexerboxes"
ALGOD_PORT = 49531
zrun:
	# ./cmd/algorand-indexer/algorand-indexer daemon -i "/Users/zeph/indexer/" -c "indexer.yml"
	./cmd/algorand-indexer/algorand-indexer daemon -i /Users/zeph/indexer/ \
	 	-P $(INDEXER_DB) \
		--pidfile /Users/zeph/indexer/algorand-indexer.pid --algod-net 127.0.0.1:8080 \
		--algod-token $(ALGOD_TOKEN) \
		-l debug


# bybye 8080: --pidfile /Users/zeph/indexer/algorand-indexer.pid --algod-net 127.0.0.1:8080

LOGFILE = boxes.log
zrunquiet:
	# ./cmd/algorand-indexer/algorand-indexer daemon -i ~/indexer/ -c "indexer.yml" > $(LOGFILE)
	./cmd/algorand-indexer/algorand-indexer daemon -i /Users/zeph/indexer/ \
	 	-P $(INDEXER_DB) \
		--pidfile /Users/zeph/indexer/algorand-indexer.pid --algod-net 127.0.0.1:$(ALGOD_PORT) \
		--algod-token $(ALGOD_TOKEN) \
		-l debug > $(LOGFILE)