-
-
Save pgunn/86c6dc47bd48bfec76364a421934a34d to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| set -e | |
| NEUTU_ENV=managed_neutu | |
| NEUTU_WRAPPER_VERSION=84 | |
| NEWT_GIST_URL=https://gist.githubusercontent.com/pgunn/86c6dc47bd48bfec76364a421934a34d/raw/ | |
| NEWT_TEMPFILE=/tmp/new_newt | |
| NEWT_PATH=$HOME/newt | |
| # General arguments for whenever we do a "conda run" | |
| CONDA_RUNARGS="--no-capture-output --live-stream" | |
| CURL=curl | |
| # If you really, really don't want to use mamba for some reason, set MAMBA=conda | |
| #MAMBA=mamba | |
| MAMBA=conda | |
| TRACE_POLICY=wasptrace.flatironinstitute.org:8000 | |
| POLICY_UUID=99aaaaa | |
| URL_USERLIST=http://$TRACE_POLICY/api/node/$POLICY_UUID/users/keys | |
| URL_USER_EXPAND=http://$TRACE_POLICY/api/node/$POLICY_UUID/users/key | |
| # Pins. | |
| # NeuTu and Neu3 release IDs don't always mirror each other, so we need separate pins. | |
| PIN_NEUTU=0.9.9 | |
| PIN_NEUTHREE=0.9.10 | |
| #PIN_LIBDVID_CPP=0.3.post118 | |
| NEUTU_DEV_ENV=managed_dev_neutu | |
| NEUTU_OLD_ENV=managed_old_neutu | |
| NEUTU_DEBUG_ENV=managed_debug_neutu | |
| DEVEL_ENV=neuclease | |
| DEVEL_JUP_ENV=neuclease_jupyter | |
| # Config stuff | |
| NEWT_CONFIG=$HOME/.newtconfig | |
| # Packages newt uses and always needs | |
| #NEWT_ALWAYS_INSTALL_BASE="mamba jq" | |
| NEWT_ALWAYS_INSTALL_BASE="jq" | |
| # Minimum acceptable version of newt is managed on a gist | |
| NEWT_MINVERSION_GIST_URL=https://gist.githubusercontent.com/pgunn/a278e73ed5c4cbaf0ff80b6e9385fa63/raw/ | |
| ##################### | |
| # newt - Flatiron NeuTu wrapper | |
| # | |
| # This is designed to run on Linux or OSX, and should handle | |
| # management and launching of NeuTu. This has a list of subcommands, | |
| # but we want to keep it simple as: | |
| # 1) shell programming is not maintainable, and | |
| # 2) Linux and OSX have few guarantees between them in terms of | |
| # package versioning or presence. | |
| # | |
| # We assume that "managed_neutu" is an environment this script owns and can | |
| # do whatever it likes with. | |
| # | |
| # Dependencies: | |
| # bash - Linux has 4.x, OSX has 3.x (because they don't like GPL) | |
| # curl - Usually present by default on Linux and OSX | |
| ################# | |
| # Find where Conda is and activate it in case the user's dotfiles didn't do that | |
| if [ -d "$HOME/miniconda3" ]; then | |
| CONDA_PATH="$HOME/miniconda3" | |
| elif [ -d "$HOME/anaconda3" ]; then | |
| CONDA_PATH="$HOME/anaconda3" | |
| elif [ -d "$HOME/miniconda2" ]; then | |
| CONDA_PATH="$HOME/miniconda2" | |
| elif [ -d "$HOME/anaconda2" ]; then | |
| CONDA_PATH="$HOME/anaconda2" | |
| elif [ -d "$HOME/opt/miniconda3" ]; then | |
| CONDA_PATH="$HOME/opt/miniconda3" | |
| elif [ -d "$HOME/opt/anaconda3" ]; then | |
| CONDA_PATH="$HOME/opt/anaconda3" | |
| elif [ -d "$HOME/programs/anaconda3" ]; then | |
| CONDA_PATH="$HOME/programs/anaconda3" | |
| elif [ ! -z "$CONDA_EXE" ]; then | |
| # Maybe actually the best way to do it? | |
| CONDA_PATH=$(dirname $CONDA_EXE)/.. | |
| else | |
| echo "Cannot find Miniconda or Anaconda in any of the standard places. Giving up." | |
| exit 1 | |
| fi | |
| # ensure that curl is installed and available in the PATH | |
| [[ ! $(type -P "$CURL") ]] && echo "$CURL is not installed and it is a prerequisite for newt" && exit 1 | |
| if [ -f "$CONDA_PATH/etc/conda_path.sh" ]; then | |
| . $CONDA_PATH/etc/conda_bash.sh | |
| elif [ -f "$CONDA_PATH/etc/profile.d/conda.sh" ]; then | |
| . $CONDA_PATH/etc/profile.d/conda.sh | |
| fi | |
| conda activate base | |
| ################ | |
| # Time check | |
| # | |
| # We would like to warn the user if they're trying to use neutu at a time when the server is either being backed up | |
| # or will be soon. Backups start at 3am. This used to be a bigger concern, but we switched to a more efficient process | |
| # sometime back so this warning is less important now | |
| HOUR_USEAST=$(TZ=:America/New_York date +%H) | |
| if [ ${HOUR_USEAST} -lt 5 ]; then | |
| echo "*****************" | |
| echo "* WARNING: the tracing server is backed up every night at 3am US-Eastern," | |
| echo "* and is briefly unavailable for a bit for that" | |
| echo "* NeuTu will not work while the tracing server is down." | |
| echo "* If you are active on it at that time, the server will disappear under you and" | |
| echo "* some of your most recent work may be lost." | |
| echo "* Caveat Operarius" | |
| echo "*****************" | |
| fi | |
| ################ | |
| # Handle usernames | |
| # | |
| # This is fairly ugly stuff that attempts to vet users trying to use NeuTu to make sure they're known, and | |
| # if their username isn't the one they're primarily using on the system, to tell NeuTu what their preferred | |
| # username is. See | |
| # https://paper.dropbox.com/doc/DVID-data-storage--AvlApU0DR_Q9m8lUqVRymd4VAg-OQjxfKbmrCqVXY9dSyVLH | |
| # for the dynamic policy side of this. | |
| # Not proud of this tr-based parsing - we just delete json characters until bash's list constructor is happy | |
| ALL_USERS=($($CURL $URL_USERLIST 2>/dev/null | tr -d "\"][" | tr ',' ' ')) | |
| if [ "${#ALL_USERS[@]}" -lt 1 ]; then | |
| echo "Could not retrieve user list; the tracing DVID server may be down (possibly nightly backups if it is after 2AM US-Eastern)" | |
| echo "Please contact pgunn if you see this message during normal times of the day" | |
| exit 1 | |
| fi | |
| # If we were on Bash 4.x I'd use an associative array for this. | |
| for u in "${ALL_USERS[@]}"; do | |
| ALTS_FOR_THIS_USER=($($CURL $URL_USER_EXPAND/$u 2>/dev/null | tr -d "\"][" | tr ',' ' ')) | |
| for a in "${ALTS_FOR_THIS_USER[@]}"; do | |
| eval "ALIAS_$a"='$u' | |
| done | |
| done | |
| # The regex max of the serialised form is probably the worst part of this part of the code. | |
| # TODO: Consider refactoring so username is always explicitly provided, to simplify any special | |
| # per-user logic | |
| EXTRA_USERNAME="" | |
| ALIAS_SYMBOL="ALIAS_$USER" | |
| if [[ " ${ALL_USERS[@]} " =~ " ${USER} " ]]; then | |
| echo "NeuTu Username: ${USER}" | |
| elif [ ! -z "${!ALIAS_SYMBOL}" ]; then | |
| echo "Looking up alias for $USER" | |
| EXTRA_USERNAME="user:${!ALIAS_SYMBOL}" | |
| echo "NeuTu Username: ${!ALIAS_SYMBOL}" | |
| else | |
| echo "Unrecognised username, please contact Pat to be added to the userlist" | |
| exit 1 | |
| fi | |
| ################ | |
| # TODO: Refactor these to reduce redundancy | |
| launch_neutu () { | |
| if [ "$(uname)" == "Darwin" ]; then | |
| # NeuTu is packaged differently on OSX to fit the bundle format for that platform | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu.app/Contents/MacOS/neutu | |
| else # Assume Linux | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu | |
| fi | |
| echo "Launching neutu as: $NEUTU_BIN $EXTRA_USERNAME" | |
| echo "Logs will have the $$ identifier" | |
| conda run $CONDA_RUNARGS -n $NEUTU_ENV $NEUTU_BIN $EXTRA_USERNAME 2>~/log-newt-$$.err >~/log-newt-$$.out | |
| } | |
| launch_neuthree () { | |
| if [ "$(uname)" == "Darwin" ]; then | |
| # Neu3 is packaged differently on OSX to fit the bundle format for that platform | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neu3.app/Contents/MacOS/neu3 | |
| else # Assume Linux | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neu3 | |
| fi | |
| echo "******* WARNING ******" | |
| echo "neu3 launched in non-guided mode will not set a cleave server" | |
| echo "If you want to use a cleave server, please use rlaunch3" | |
| echo "**********************" | |
| echo "Logs will have the $$ identifier" | |
| conda run $CONDA_RUNARGS -n $NEUTU_ENV $NEUTU_BIN $EXTRA_USERNAME 2>~/log-newt-$$.err >~/log-newt-$$.out | |
| } | |
| launch_guided_local_neutu() { | |
| generate_neutu_localconfig_from_argtwo_and_argthree | |
| echo "Generated configs" | |
| if [ "$(uname)" == "Darwin" ]; then | |
| # NeuTu is packaged differently on OSX to fit the bundle format for that platform | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu.app/Contents/MacOS/neutu | |
| else # Assume Linux | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu | |
| fi | |
| echo "Launching neutu as: $NEUTU_BIN $EXTRA_USERNAME --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN" | |
| echo "Logs will have the $$ identifier" | |
| conda run $CONDA_RUNARGS -n $NEUTU_ENV $NEUTU_BIN --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN $EXTRA_USERNAME 2>~/log-newt-$$.err >~/log-newt-$$.out | |
| } | |
| launch_guided_localish_neutu() { | |
| generate_neutu_localishconfig_from_args | |
| echo "Generated configs" | |
| if [ "$(uname)" == "Darwin" ]; then | |
| # NeuTu is packaged differently on OSX to fit the bundle format for that platform | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu.app/Contents/MacOS/neutu | |
| else # Assume Linux | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu | |
| fi | |
| echo "Launching neutu as: $NEUTU_BIN $EXTRA_USERNAME --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN" | |
| echo "Logs will have the $$ identifier" | |
| conda run $CONDA_RUNARGS -n $NEUTU_ENV $NEUTU_BIN --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN $EXTRA_USERNAME 2>~/log-newt-$$.err >~/log-newt-$$.out | |
| } | |
| launch_guided_devel_neutu() { | |
| generate_neutu_config_from_argtwo | |
| echo "Generated configs" | |
| if [ "$(uname)" == "Darwin" ]; then | |
| # NeuTu is packaged differently on OSX to fit the bundle format for that platform | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_DEV_ENV/bin/neutu.app/Contents/MacOS/neutu | |
| else # Assume Linux | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_DEV_ENV/bin/neutu | |
| fi | |
| echo "Launching neutu as: $NEUTU_BIN $EXTRA_USERNAME --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN" | |
| echo "Logs will have the $$ identifier" | |
| conda run $CONDA_RUNARGS -n $NEUTU_DEV_ENV $NEUTU_BIN --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN $EXTRA_USERNAME 2>~/log-newt-$$.err >~/log-newt-$$.out | |
| } | |
| launch_guided_debug_neutu() { | |
| generate_neutu_config_from_argtwo | |
| echo "Generated configs" | |
| if [ "$(uname)" == "Darwin" ]; then | |
| # NeuTu is packaged differently on OSX to fit the bundle format for that platform | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_DEBUG_ENV/bin/neutu_d.app/Contents/MacOS/neutu_d | |
| else # Assume Linux | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_DEBUG_ENV/bin/neutu_d | |
| fi | |
| echo "Launching neutu as: $NEUTU_BIN $EXTRA_USERNAME --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN" | |
| echo "Logs will have the $$ identifier" | |
| conda run $CONDA_RUNARGS -n $NEUTU_DEBUG_ENV $NEUTU_BIN --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN $EXTRA_USERNAME 2>~/log-newt-$$.err >~/log-newt-$$.out | |
| } | |
| launch_guided_neutu () { | |
| generate_neutu_config_from_argtwo | |
| echo "Generated configs" | |
| if [ "$(uname)" == "Darwin" ]; then | |
| # NeuTu is packaged differently on OSX to fit the bundle format for that platform | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu.app/Contents/MacOS/neutu | |
| else # Assume Linux | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu | |
| fi | |
| echo "Launching neutu as: $NEUTU_BIN $EXTRA_USERNAME --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN" | |
| echo "Logs will have the $$ identifier" | |
| conda run $CONDA_RUNARGS -n $NEUTU_ENV $NEUTU_BIN --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN $EXTRA_USERNAME 2>~/log-newt-$$.err >~/log-newt-$$.out | |
| } | |
| launch_guided_neuthree () { | |
| # TODO: Also retrieve the cleave server and set env vars appropriately | |
| generate_neutu_config_from_argtwo | |
| URL_GET_CLEAVE_SERVER=http://$TRACE_POLICY/api/node/$AUTOREPO:main/fi_info/key/cleave_server | |
| set +e | |
| POTENTIAL_CLEAVE_SERVER=$($CURL $URL_GET_CLEAVE_SERVER 2>/dev/null | tr -d "\"" | grep -v "could not find" | grep -v "not found") | |
| set -e | |
| if [ -z "$POTENTIAL_CLEAVE_SERVER" ]; then | |
| echo "Could not find cleave server for repo $AUTOREPO" | |
| else | |
| export NEU3_CLEAVE_SERVER=http://$POTENTIAL_CLEAVE_SERVER | |
| echo "Set cleave server to http://$POTENTIAL_CLEAVE_SERVER" | |
| fi | |
| echo "Generated configs" | |
| if [ "$(uname)" == "Darwin" ]; then | |
| # Neu3 is packaged differently on OSX to fit the bundle format for that platform | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neu3.app/Contents/MacOS/neu3 | |
| else # Assume Linux | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neu3 | |
| fi | |
| echo "Logs will have the $$ identifier" | |
| conda run $CONDA_RUNARGS -n $NEUTU_ENV $NEUTU_BIN --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN $EXTRA_USERNAME 2>~/log-newt-$$.err >~/log-newt-$$.out | |
| } | |
| launch_viewmode_neutu() { | |
| # args: | |
| # $1 is specimen | |
| generate_view_config $1 | |
| echo "Generated configs" | |
| if [ "$(uname)" == "Darwin" ]; then | |
| # NeuTu is packaged differently on OSX to fit the bundle format for that platform | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu.app/Contents/MacOS/neutu | |
| else # Assume Linux | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu | |
| fi | |
| echo "Launching neutu as: $NEUTU_BIN $EXTRA_USERNAME --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN" | |
| echo "Logs will have the $$ identifier" | |
| conda run $CONDA_RUNARGS -n $NEUTU_ENV $NEUTU_BIN --load config:/tmp/neu_${USER}/neu_wrapper.json dname:AUTOGEN $EXTRA_USERNAME 2>~/log-newt-$$.err >~/log-newt-$$.out | |
| } | |
| launch_the_skeletons() { | |
| # args: | |
| # $1 is the repospec | |
| # $2 is either: | |
| # numeric value of the bodyid we want to skeletonise | |
| # text file ending in .txt containing multiple bodyids to skeletonise | |
| # (neutu handles all the difference between those uses, we just pass it along and hope it does the right thing) | |
| # | |
| # With this usage, we don't need to generate any config files; all the config information is passed to neutu from the CLI; we just need to | |
| # generate that config string and arrange for neutu to be launched. | |
| REPO=$1 | |
| BODYSPEC=$2 | |
| if [ "$(uname)" == "Darwin" ]; then | |
| # NeuTu is packaged differently on OSX to fit the bundle format for that platform | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu.app/Contents/MacOS/neutu | |
| else # Assume Linux | |
| NEUTU_BIN=$CONDA_PATH/envs/$NEUTU_ENV/bin/neutu | |
| fi | |
| # TODO: This chunk of code exists elsewhere in this script and should be genericised | |
| TRACE_ONE='wasptrace.flatironinstitute.org' # Look here first | |
| TRACE_TWO='wasptrace2.flatironinstitute.org' # If not there, then here | |
| TRACE_THR='wasptrace3.flatironinstitute.org' # If not there, then here | |
| TRACING_SERVER=$TRACE_ONE | |
| if [ $(repo_exists_on_server $TRACING_SERVER $REPO) != '200' ]; then | |
| echo "Trace fallback" | |
| TRACING_SERVER=$TRACE_TWO | |
| if [ $(repo_exists_on_server $TRACING_SERVER $REPO) != '200' ]; then | |
| TRACING_SERVER=$TRACE_THR | |
| if [ $(repo_exists_on_server $TRACING_SERVER $REPO) != '200' ]; then | |
| echo "Trace fallback failed: $TRACING_SERVER failed or does not have $REPO" | |
| exit 1 | |
| fi | |
| fi | |
| fi | |
| SKELCONFIG="http://$TRACING_SERVER?uuid=$REPO:main\&segmentation=labels" | |
| echo "Launching skeletonization mode with IDs=$BODYSPEC CONFIG=$SKELCONFIG" | |
| if [ ! -d "/tmp/neu_{$USER}" ]; then | |
| mkdir "/tmp/neu_{$USER}" | |
| fi | |
| SKELSCRIPT=/tmp/neu_${USER}/skel_launcher.sh | |
| echo "#!/bin/bash" > $SKELSCRIPT | |
| echo "neutu --command --skeletonize --bodyid $BODYSPEC $SKELCONFIG" >> $SKELSCRIPT | |
| chmod 755 $SKELSCRIPT | |
| conda run $CONDA_RUNARGS -n $NEUTU_ENV $SKELSCRIPT | |
| } | |
| ################# | |
| # Config generation | |
| # | |
| # TODO: A lot of this code is duplicated and should be condensed, maybe by changing some of the code blocks into their own functions | |
| # Real full unification of the functions would be hard (different arguments, which are hard to do conditionally in a non-confusing | |
| # way, but if the config generation uses helper functions each one could be quite short and the differences would be more apparent. | |
| generate_neutu_config_from_argtwo () { | |
| # This is the main codepath for launching neutu in newt | |
| # generate the two stub json files needed to launch neutu/neu3 against that repo | |
| # TODO If it took args, this would be sensible: | |
| # $1 - Was a global called AUTOREPO, should just be REPO or UUID or similar | |
| mkdir -p /tmp/neu_${USER} | |
| rm -f /tmp/neu_${USER}/neu_wrapper.json /tmp/neu_${USER}/neu_gen.json | |
| cat > /tmp/neu_${USER}/neu_wrapper.json <<EOCFGWRAPPER | |
| { "flyem": ["/tmp/neu_${USER}/neu_gen.json"] } | |
| EOCFGWRAPPER | |
| # Now with the easy part out of the way, we need to generate a correct neutu config for the | |
| # repo living in $1. Next we pull the specimen from the repo identifier; with that, | |
| # the rest we can hardcode because we have only two specimens and most of the config for both is | |
| # static. | |
| check_if_command_exists jq | |
| SPECIMEN=$(get_specimen_id_for_repo $AUTOREPO) | |
| if [ "$SPECIMEN" == "FAIL" ]; then | |
| echo "Repo $AUTOREPO was found but not marked with manage_dvid_seg; could not identify specimen" | |
| exit 1 | |
| fi | |
| echo "Working with specimen $SPECIMEN" | |
| resolve_config | |
| if [ "$SPECIMEN" == "wasp1" ]; then | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp2" ]; then | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp3" ]; then | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp2a" ]; then | |
| # This has always been a special case | |
| GRAYSCALE="wasp2a" | |
| IMAGEUUID="d3e5db172" | |
| IMAGESERVER="waspem-dvid.flatironinstitute.org" | |
| else | |
| echo "Error: Failed to get specimen data for repo $AUTOREPO:main" | |
| exit 1 | |
| fi | |
| if [ "$LAUNCH_MODE" == "normal" ]; then | |
| USE_UUID="$AUTOREPO:main" | |
| elif [ "$LAUNCH_MODE" == "exact" ]; then | |
| USE_UUID="$AUTOREPO" | |
| else | |
| echo "Unknown launch mode" | |
| exit 1 | |
| fi | |
| TRACE_ONE='wasptrace.flatironinstitute.org' # Look here first | |
| TRACE_TWO='wasptrace2.flatironinstitute.org' # If not there, then here | |
| TRACE_THR='wasptrace3.flatironinstitute.org' # If not there, then here | |
| TRACING_SERVER=$TRACE_ONE | |
| # time for sanity checks and fallbacks | |
| if [ $(repo_exists_on_server $IMAGESERVER $IMAGEUUID) != '200' ]; then | |
| echo "Image server $IMAGESERVER may be down? NeuTu will not function" | |
| exit 1 | |
| fi | |
| if [ $(repo_exists_on_server $TRACING_SERVER $USE_UUID) != '200' ]; then | |
| echo "Trace fallback" | |
| TRACING_SERVER=$TRACE_TWO | |
| if [ $(repo_exists_on_server $TRACING_SERVER $USE_UUID) != '200' ]; then | |
| TRACING_SERVER=$TRACE_THR | |
| if [ $(repo_exists_on_server $TRACING_SERVER $USE_UUID) != '200' ]; then | |
| echo "Trace fallback failed: $TRACING_SERVER failed or does not have $USE_UUID" | |
| exit 1 | |
| fi | |
| fi | |
| fi | |
| # Remember: When you don't quote your HEREDOC end marker, you get string interpolation. | |
| cat > /tmp/neu_${USER}/neu_gen.json <<EOCFGNEUGEN | |
| { | |
| "dvid repo": [{"gray_scale": "$GRAYSCALE","name": "AUTOGEN", "roi_list": [], "roi": "${ROI+$ROI}", "multires_tile": "", "port": 8000, "synapse": "synapses", "host": "$TRACING_SERVER", "body_label": "labels", "uuid": "$USE_UUID", "@source": { "multires_tile": { "port": 80, "host": "$IMAGESERVER", "uuid": "$IMAGEUUID" }, "gray_scale": { "port": 80, "host": "$IMAGESERVER", "uuid": "$IMAGEUUID" } }, "label_max_zoom": 0, "background": 255, "label_block": "labels", "multires_tile_config": { "$TILES": { "low_quality": false } }, "grayscale_max_zoom": 0, "proofreading": true, "supervised": true, "librarian": "$TRACING_SERVER:$LIBPORT", "default": false}] | |
| } | |
| EOCFGNEUGEN | |
| } | |
| generate_neutu_localconfig_from_argtwo_and_argthree () { | |
| # Does not talk to the policy repo the same way, because this is for segmentations that run on localhost | |
| # generate the two stub json files needed to launch neutu/neu3 against that repo | |
| # TODO If it took args, this would be sensible: | |
| # $1 - Was a global called AUTOREPO, should just be REPO or UUID or similar | |
| # $2 - Was a global called SPEC | |
| mkdir -p /tmp/neu_${USER} | |
| rm -f /tmp/neu_${USER}/neu_wrapper.json /tmp/neu_${USER}/neu_gen.json | |
| cat > /tmp/neu_${USER}/neu_wrapper.json <<EOLCFGWRAPPER | |
| { "flyem": ["/tmp/neu_${USER}/neu_gen.json"] } | |
| EOLCFGWRAPPER | |
| # Now with the easy part out of the way, we need to generate a correct neutu config for the | |
| # repo living in $1. Next we pull the specimen from the repo identifier; with that, | |
| # the rest we can hardcode because we have only two specimens and most of the config for both is | |
| # static. | |
| check_if_command_exists jq | |
| echo "Building localconfig for specimen [$SPEC]" | |
| SPECIMEN=$SPEC | |
| echo "Working with specimen $SPECIMEN" | |
| resolve_config | |
| if [ "$SPECIMEN" == "wasp1" ]; then | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp2" ]; then | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp3" ]; then | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp2a" ]; then | |
| GRAYSCALE="wasp2a" | |
| IMAGEUUID="d3e5db172" | |
| IMAGESERVER="waspem-dvid.flatironinstitute.org" | |
| else | |
| echo "Error: Failed to get specimen data for repo $AUTOREPO:main" | |
| exit 1 | |
| fi | |
| if [ "$LAUNCH_MODE" == "normal" ]; then | |
| USE_UUID="$AUTOREPO:main" | |
| elif [ "$LAUNCH_MODE" == "exact" ]; then | |
| USE_UUID="$AUTOREPO" | |
| else | |
| echo "Unknown launch mode" | |
| exit 1 | |
| fi | |
| # Remember: When you don't quote your HEREDOC end marker, you get string interpolation. | |
| cat > /tmp/neu_${USER}/neu_gen.json <<EOCFGNEULGEN | |
| { | |
| "dvid repo": [{"gray_scale": "$GRAYSCALE","name": "AUTOGEN", "roi_list": [], "roi": "${ROI+$ROI}", "multires_tile": "", "port": 8000, "synapse": "synapses", "host": "localhost:8000", "body_label": "labels", "uuid": "$USE_UUID", "@source": { "multires_tile": { "port": 80, "host": "$IMAGESERVER", "uuid": "$IMAGEUUID" }, "gray_scale": { "port": 80, "host": "$IMAGESERVER", "uuid": "$IMAGEUUID" } }, "label_max_zoom": 0, "background": 255, "label_block": "labels", "multires_tile_config": { "$TILES": { "low_quality": false } }, "grayscale_max_zoom": 0, "proofreading": true, "supervised": true, "librarian": "", "default": false}] | |
| } | |
| EOCFGNEULGEN | |
| } | |
| generate_neutu_localishconfig_from_args () { | |
| # This is used to launch neutu in a configuration used for testing segmentation running on a system | |
| # that's not our normal segmentation hosts. It's very rarely useful. | |
| # generate the two stub json files needed to launch neutu/neu3 against that repo | |
| # TODO If it took args, this would be sensible (calling code needs to flip order around): | |
| # $1 - Was a global called AUTOREPO, should just be REPO or UUID or similar | |
| # $2 - Was a global called SEG_HOST, holding hostname of alternate segmentation server | |
| # $3 - Was a global called SPEC, holding specimen id | |
| mkdir -p /tmp/neu_${USER} | |
| rm -f /tmp/neu_${USER}/neu_wrapper.json /tmp/neu_${USER}/neu_gen.json | |
| cat > /tmp/neu_${USER}/neu_wrapper.json <<EOLCFGWRAPPER | |
| { "flyem": ["/tmp/neu_${USER}/neu_gen.json"] } | |
| EOLCFGWRAPPER | |
| # Now with the easy part out of the way, we need to generate a correct neutu config for the | |
| # repo living in $1. Next we pull the specimen from the repo identifier; with that, | |
| # the rest we can hardcode because we have only two specimens and most of the config for both is | |
| # static. | |
| check_if_command_exists jq | |
| echo "Building localconfig for specimen [$SPEC]" | |
| SPECIMEN=$SPEC | |
| echo "Working with specimen $SPECIMEN" | |
| resolve_config | |
| if [ "$SPECIMEN" == "wasp1" ]; then | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp2" ]; then | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp3" ]; then | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp2a" ]; then | |
| GRAYSCALE="wasp2a" | |
| IMAGEUUID="d3e5db172" | |
| IMAGESERVER="waspem-dvid.flatironinstitute.org" | |
| else | |
| echo "Error: Failed to get specimen data for repo $AUTOREPO:main" | |
| exit 1 | |
| fi | |
| if [ "$LAUNCH_MODE" == "normal" ]; then | |
| USE_UUID="$AUTOREPO:main" | |
| elif [ "$LAUNCH_MODE" == "exact" ]; then | |
| USE_UUID="$AUTOREPO" | |
| else | |
| echo "Unknown launch mode" | |
| exit 1 | |
| fi | |
| # Remember: When you don't quote your HEREDOC end marker, you get string interpolation. | |
| cat > /tmp/neu_${USER}/neu_gen.json <<EOCFGNEULGEN | |
| { | |
| "dvid repo": [{"gray_scale": "$GRAYSCALE","name": "AUTOGEN", "roi_list": [], "roi": "${ROI+$ROI}", "multires_tile": "", "port": 8000, "synapse": "synapses", "host": "$SEG_HOST:8000", "body_label": "labels", "uuid": "$USE_UUID", "@source": { "multires_tile": { "port": 80, "host": "$IMAGESERVER", "uuid": "$IMAGEUUID" }, "gray_scale": { "port": 80, "host": "$IMAGESERVER", "uuid": "$IMAGEUUID" } }, "label_max_zoom": 0, "background": 255, "label_block": "labels", "multires_tile_config": { "$TILES": { "low_quality": false } }, "grayscale_max_zoom": 0, "proofreading": true, "supervised": true, "librarian": "", "default": false}] | |
| } | |
| EOCFGNEULGEN | |
| } | |
| generate_view_config () { | |
| # This is used for the "./newt view wasp3" | |
| # command and similar, used to just view image stacks with no (real) segmentation loaded. | |
| # | |
| # generate the two stub json files needed to launch neutu/neu3 against that repo | |
| # args: | |
| # $1 is specimen | |
| SPECIMEN=$1 | |
| mkdir -p /tmp/neu_${USER} | |
| rm -f /tmp/neu_${USER}/neu_wrapper.json /tmp/neu_${USER}/neu_gen.json | |
| cat > /tmp/neu_${USER}/neu_wrapper.json <<EOCFGWRAPPER | |
| { "flyem": ["/tmp/neu_${USER}/neu_gen.json"] } | |
| EOCFGWRAPPER | |
| # Now with the easy part out of the way, we need to generate a correct neutu config for the | |
| # repo living in $1. Next we pull the specimen from the repo identifier; with that, | |
| # the rest we can hardcode because we have only two specimens and most of the config for both is | |
| # static. | |
| check_if_command_exists jq | |
| echo "Working with specimen $SPECIMEN" | |
| resolve_config | |
| echo "Building view mode for specimen $SPECIMEN" | |
| if [ "$SPECIMEN" == "wasp1" ]; then | |
| USE_UUID="98aaaa" | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP1_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp2" ]; then | |
| USE_UUID="98bbbb" | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP2_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp3" ]; then | |
| USE_UUID="98dddd" | |
| GRAYSCALE=$(instance_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| IMAGEUUID=$(repo_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| IMAGESERVER=$(server_for_specimen_imageset $SPECIMEN $WASP3_IMGSET) | |
| elif [ "$SPECIMEN" == "wasp2a" ]; then | |
| USE_UUID="98cccc" | |
| GRAYSCALE="wasp2a" | |
| IMAGEUUID="d3e5db172" | |
| IMAGESERVER="waspem-dvid.flatironinstitute.org" | |
| else | |
| echo "Error: Failed to get specimen data for view mode" | |
| exit 1 | |
| fi | |
| # Remember: When you don't quote your HEREDOC end marker, you get string interpolation. | |
| cat > /tmp/neu_${USER}/neu_gen.json <<EOCFGNEUGEN | |
| { | |
| "dvid repo": [{"gray_scale": "$GRAYSCALE","name": "AUTOGEN", "roi_list": [], "roi": "${ROI+$ROI}","multires_tile": "", "port": 8000, "synapse": "synapses", "host": "wasptrace.flatironinstitute.org", "body_label": "labels", "uuid": "$USE_UUID", "@source": { "multires_tile": { "port": 80, "host": "$IMAGESERVER", "uuid": "$IMAGEUUID" }, "gray_scale": { "port": 80, "host": "$IMAGESERVER", "uuid": "$IMAGEUUID" } }, "label_max_zoom": 0, "background": 255, "label_block": "labels", "grayscale_max_zoom": 0, "proofreading": true, "supervised": true, "librarian": "wasptrace.flatironinstitute.org:$LIBPORT", "default": false}] | |
| } | |
| EOCFGNEUGEN | |
| } | |
| ###################### | |
| # The following are conceptually similar to fivol.py:get_imageconfigs_for_specimen() | |
| # | |
| # They're less general because shell programming is more awkward about data representations | |
| # available (shell programming is terrible, particularly if you're trying to be portable) | |
| summarise_imagesets_for_specimen() { | |
| # This was originally intended to return the imagesets for a specimen in a convenient format | |
| # for later parsing, but bash is not good at that kind of thing. Recycled into something that will | |
| # tell users what their imageset options are for a given specimen, for use by the config argument with | |
| # no subarguments. | |
| SPECIMEN=$1 | |
| URL_ISETLIST=http://$TRACE_POLICY/api/node/$POLICY_UUID/specimen_$SPECIMEN/keys | |
| IMAGESETS=($($CURL $URL_ISETLIST 2>/dev/null | tr -d "\"][" | tr ',' ' ')) | |
| if [ "${#IMAGESETS[@]}" -lt 1 ]; then | |
| echo "I was unable to find any imagesets for the requested specimen, which should be something like wasp1, wasp2, wasp2a, or similar" | |
| exit 1 | |
| fi | |
| printf "\t\t%s\n" "${IMAGESETS[@]}" | |
| } | |
| repo_for_specimen_imageset() { | |
| SPECIMEN=$1 | |
| IMAGESET=$2 | |
| URL_ISETINFO=http://$TRACE_POLICY/api/node/$POLICY_UUID/specimen_$SPECIMEN/key/$IMAGESET | |
| REPO=($($CURL $URL_ISETINFO 2>/dev/null | jq '.repo' | tr -d "\"][" )) | |
| echo $REPO | |
| } | |
| server_for_specimen_imageset() { | |
| # The port is in there too, but I'm comfortable hardcoding that for now and "cut"ting it out | |
| SPECIMEN=$1 | |
| IMAGESET=$2 | |
| URL_ISETINFO=http://$TRACE_POLICY/api/node/$POLICY_UUID/specimen_$SPECIMEN/key/$IMAGESET | |
| # the retrieval gets something like "http://waspem-dvid.flatironinstitute.org:8000" | |
| # so we need to get past the slashes and before the colon | |
| SAP=($($CURL $URL_ISETINFO 2>/dev/null | jq '.server' | cut -d/ -f3 | cut -d: -f1 )) | |
| echo $SAP | |
| } | |
| instance_for_specimen_imageset() { | |
| SPECIMEN=$1 | |
| IMAGESET=$2 | |
| URL_ISETINFO=http://$TRACE_POLICY/api/node/$POLICY_UUID/specimen_$SPECIMEN/key/$IMAGESET | |
| INST=($($CURL $URL_ISETINFO 2>/dev/null | jq '.instance' | tr -d '"' )) | |
| echo $INST | |
| } | |
| ###################### | |
| # Checking DVID servers for things | |
| # | |
| # This is used for fallbacks and sanity checks | |
| repo_exists_on_server() { | |
| # If you hit the /info endpoint of a real repo, you get a 200. Otherwise you get a 400. This is a reliable way to see if a repo is present on a DVID server (so far). | |
| SERVER=$1 | |
| REPO=$2 | |
| URL_CHECK=http://$SERVER:8000/api/repo/$REPO/info | |
| RESPONSE=($($CURL -s -o /dev/null -I -w "%{http_code}" $URL_CHECK)) | |
| echo $RESPONSE | |
| } | |
| get_specimen_id_for_repo() { | |
| # Reliably, through fallback, identify the specimen for a repo or give the user a good error message | |
| REPO=$1 | |
| TRY1=wasptrace.flatironinstitute.org # FIXME move this out to a shared list | |
| TRY2=wasptrace2.flatironinstitute.org | |
| if [ $(repo_exists_on_server $TRY1 $REPO) == '200' ]; then # first server, like most repos | |
| if [ "$LAUNCH_MODE" == "exact" ]; then | |
| URL_INFO_SPECIMEN=http://$TRY1:8000/api/node/$AUTOREPO/fi_info/key/specimen | |
| else | |
| URL_INFO_SPECIMEN=http://$TRY1:8000/api/node/$AUTOREPO:main/fi_info/key/specimen | |
| fi | |
| SPECIMEN=$($CURL $URL_INFO_SPECIMEN 2>/dev/null | tr -d "\"") | |
| echo $SPECIMEN | |
| elif [ $(repo_exists_on_server $TRY2 $REPO) == '200' ]; then # second server, newer repos | |
| if [ "$LAUNCH_MODE" == "exact" ]; then | |
| URL_INFO_SPECIMEN=http://$TRY2:8000/api/node/$AUTOREPO/fi_info/key/specimen | |
| else | |
| URL_INFO_SPECIMEN=http://$TRY2:8000/api/node/$AUTOREPO:main/fi_info/key/specimen | |
| fi | |
| SPECIMEN=$($CURL $URL_INFO_SPECIMEN 2>/dev/null | tr -d "\"") | |
| echo $SPECIMEN | |
| else | |
| echo "FAIL" # Because of bash call semantics, this isn't the right place to spit out a good error message; caller needs to do that. | |
| fi | |
| } | |
| ###################### | |
| # Config and defaults management | |
| # | |
| # Functions here manage having some builtin default settings that can be overwritten by a config | |
| # file, as well as management of that config file. The ideal is that users don't themselves edit | |
| # that file (and it might not be present). A newt config won't necessarily have an entry for all | |
| # possible configs. | |
| # | |
| # A lot of the settings here are half-hardcoded and ugly because bash doesn't offer good data structures | |
| # to hold things. If we could depend on Bash4, we'd have more options, but OSX will be on Bash3 | |
| # for the long haul (license issues). | |
| # The port that the Librarian runs on is also unfortunately commonly a printer port, and so | |
| # often is blocked by networking groups. As local configuration, we also expose the librarian on port 8443, | |
| # which is less likely to be blocked, and we offer a user-tweakable config | |
| DEFAULT_LIBPORT=9100 | |
| DEFAULT_WASP1_IMGSET=sch1 | |
| DEFAULT_WASP2_IMGSET=tyk1 | |
| DEFAULT_WASP3_IMGSET=three_clahe | |
| OVERRODE_WASP1_IMGSET=false | |
| OVERRODE_WASP2_IMGSET=false | |
| OVERRODE_WASP3_IMGSET=false | |
| OVERRODE_LIBPORT=false | |
| resolve_config() { | |
| # This reads a config file and resolves any overrides with the defaults, presenting the final versions | |
| # of variables to other parts of the codebase. | |
| LIBPORT=$DEFAULT_LIBPORT | |
| WASP1_IMGSET=$DEFAULT_WASP1_IMGSET | |
| WASP2_IMGSET=$DEFAULT_WASP2_IMGSET | |
| WASP3_IMGSET=$DEFAULT_WASP3_IMGSET | |
| if [ ! -f $NEWT_CONFIG ]; then | |
| echo "Using default newt config" | |
| else | |
| echo "Parsing newt config to override some defaults" | |
| # Next we'll temporarily turn off "exit on error" so we can handle errors better | |
| set +e | |
| # Begin the parsing. It should be easy to add new parsing this way. | |
| # grep exits with nonzero return code if it can't find its search string in the file. | |
| # we use that to look for an entry to decide if we want to override the current value. | |
| grep LIBPORT= $NEWT_CONFIG >/dev/null | |
| if [ $? -eq 0 ]; then | |
| echo "Overriding librarian port" | |
| LIBPORT=($(grep LIBPORT= $NEWT_CONFIG 2>/dev/null | cut -d= -f2)) | |
| OVERRODE_LIBPORT=true | |
| fi | |
| grep WASP1_IMGSET= $NEWT_CONFIG >/dev/null | |
| if [ $? -eq 0 ]; then | |
| WASP1_IMGSET=($(grep WASP1_IMGSET= $NEWT_CONFIG 2>/dev/null | cut -d= -f2)) | |
| OVERRODE_WASP1_IMGSET=true | |
| fi | |
| grep WASP2_IMGSET= $NEWT_CONFIG >/dev/null | |
| if [ $? -eq 0 ]; then | |
| WASP2_IMGSET=($(grep WASP2_IMGSET= $NEWT_CONFIG 2>/dev/null | cut -d= -f2)) | |
| OVERRODE_WASP2_IMGSET=true | |
| fi | |
| grep WASP3_IMGSET= $NEWT_CONFIG >/dev/null | |
| if [ $? -eq 0 ]; then | |
| WASP3_IMGSET=($(grep WASP3_IMGSET= $NEWT_CONFIG 2>/dev/null | cut -d= -f2)) | |
| OVERRODE_WASP3_IMGSET=true | |
| fi | |
| # Restore standard strict error handling | |
| set -e | |
| fi | |
| } | |
| set_config_item() { | |
| # This creates a config file if it does not already exist, or rewrites it if it does so it has the requested | |
| # config entry and value. It removes any existing entry that would set the value that it wants to set. | |
| # The mechanism to do this is the OVERRODE_* set of variables. | |
| # | |
| # You *must* call resolve_config() before using this function. | |
| # | |
| # This is a bit clunky but it does the job with what bash can offer. | |
| NEWKEY=$1 | |
| NEWVAL=$2 | |
| if [ "$NEWKEY" = "WASP1_IMGSET" ]; then | |
| OVERRODE_WASP1_IMGSET=true | |
| WASP1_IMGSET=$NEWVAL | |
| elif [ "$NEWKEY" = "WASP2_IMGSET" ]; then | |
| OVERRODE_WASP2_IMGSET=true | |
| WASP2_IMGSET=$NEWVAL | |
| elif [ "$NEWKEY" = "WASP3_IMGSET" ]; then | |
| OVERRODE_WASP3_IMGSET=true | |
| WASP3_IMGSET=$NEWVAL | |
| elif [ "$NEWKEY" = "LIBPORT" ]; then | |
| echo "Updating LIBPORT in config" | |
| OVERRODE_LIBPORT=true | |
| LIBPORT=$NEWVAL | |
| else | |
| echo "Failed to reset unknown value $NEWKEY" | |
| exit 1 | |
| fi | |
| rm -f $NEWT_CONFIG | |
| if [ "$OVERRODE_WASP1_IMGSET" = true ]; then | |
| echo "WASP1_IMGSET=$WASP1_IMGSET" >> $NEWT_CONFIG | |
| fi | |
| if [ "$OVERRODE_WASP2_IMGSET" = true ]; then | |
| echo "WASP2_IMGSET=$WASP2_IMGSET" >> $NEWT_CONFIG | |
| fi | |
| if [ "$OVERRODE_WASP3_IMGSET" = true ]; then | |
| echo "WASP3_IMGSET=$WASP3_IMGSET" >> $NEWT_CONFIG | |
| fi | |
| if [ "$OVERRODE_LIBPORT" = true ]; then | |
| echo "LIBPORT=$LIBPORT" >> $NEWT_CONFIG | |
| fi | |
| } | |
| display_all_config_and_options() { | |
| # Display all known config items, and what's set as well as what the other options are. | |
| # You must call resolve_config() before using this function | |
| if [ "$OVERRODE_WASP1_IMGSET" = true ]; then | |
| echo " WASP1_IMGSET (overridden)=$WASP1_IMGSET" | |
| else | |
| echo " WASP1_IMGSET (default)=$WASP1_IMGSET" | |
| fi | |
| echo " Options:" | |
| summarise_imagesets_for_specimen wasp1 | |
| if [ "$OVERRODE_WASP2_IMGSET" = true ]; then | |
| echo " WASP2_IMGSET (overridden)=$WASP2_IMGSET" | |
| else | |
| echo " WASP2_IMGSET (default)=$WASP2_IMGSET" | |
| fi | |
| echo " Options:" | |
| summarise_imagesets_for_specimen wasp2 | |
| if [ "$OVERRODE_WASP3_IMGSET" = true ]; then | |
| echo " WASP3_IMGSET (overridden)=$WASP3_IMGSET" | |
| else | |
| echo " WASP3_IMGSET (default)=$WASP3_IMGSET" | |
| fi | |
| echo " Options:" | |
| summarise_imagesets_for_specimen wasp3 | |
| if [ "$OVERRODE_LIBPORT" = true ]; then | |
| echo " LIBPORT (overridden)=$LIBPORT" | |
| else | |
| echo " LIBPORT (default)=$LIBPORT" | |
| fi | |
| echo " Options:" | |
| echo " Any valid numeric TCP port value (but must match server config). Default is $DEFAULT_LIBPORT but 8443 is also supported." | |
| } | |
| ###################### | |
| # misc/general | |
| check_if_command_exists() { | |
| # Please don't use this with arguments, as it might do something dangerous | |
| COMMAND=$1 | |
| set +e | |
| $1 --help 2>&1 >/dev/null | |
| if [ $? -ne 0 ]; then | |
| echo "newt needs the $1 utility but could not find it. Please run ./newt update-prereqs" | |
| exit 1 | |
| fi | |
| set -e | |
| } | |
| ###################### | |
| # Main command handler | |
| echo "This is newt version $NEUTU_WRAPPER_VERSION" | |
| if [ "$1" == "launch" ]; then | |
| export USE_SEQUENCER=yes | |
| launch_neutu | |
| elif [ "$1" == "rlaunch" ]; then | |
| ################# | |
| # Check minimum version | |
| # | |
| # We now check a gist to make sure we're current enough on our "newt" version (not neutu version) | |
| # to start up, and refuse to start if we're not. | |
| # This obviously won't help if someone is using a version of newt that doesn't have this code, but | |
| # we can at least make the future a better place. | |
| # | |
| # This will let us make breaking changes to the configuration more safely in the future. | |
| # | |
| # I would have liked to have had this check be elsewhere, but it's important that it not impact | |
| # newt's ability to update itself! | |
| NEWT_MIN_VERSION=$($CURL $NEWT_MINVERSION_GIST_URL 2>/dev/null) | |
| if [ "$NEUTU_WRAPPER_VERSION" -lt "$NEWT_MIN_VERSION" ]; then | |
| echo "\tYour version of newt (not NeuTu, just newt) is too old." | |
| echo "\tPlease update it with:" | |
| echo "./newt update-newt" | |
| echo "\tYou do NOT need to do a ./newt update afterwards" | |
| exit 1 | |
| fi | |
| export USE_SEQUENCER=yes | |
| # BEGIN ARGUMENT PARSING | |
| if [ "$#" -eq 3 ]; then | |
| ROI=$3 | |
| elif [ "$#" -lt 2 ]; then | |
| echo "Usage: ./newt rlaunch 16d7 OPTIONAL_ROI_NAME" | |
| exit 1 | |
| elif [ "$#" -gt 3 ]; then | |
| echo "Usage: ./newt rlaunch 16d7 OPTIONAL_ROI_NAME" | |
| exit 1 | |
| fi | |
| if [ -z "$2" ]; then | |
| echo "FATAL: Could not parse repo while generating configs" | |
| exit 1 | |
| else | |
| AUTOREPO=$2 | |
| fi | |
| # END ARGUMENT PARSING | |
| LAUNCH_MODE="normal" | |
| launch_guided_neutu | |
| elif [ "$1" == "rlaunch-exact" ]; then | |
| # This option is like rlaunch, but the repo identifier is parsed literally, | |
| # allowing the user to supply a bare UUID without :main being appended, or | |
| # allowing them to add another label for UUID resolution | |
| export USE_SEQUENCER=yes | |
| # BEGIN ARGUMENT PARSING | |
| if [ "$#" -eq 3 ]; then | |
| ROI=$3 | |
| elif [ "$#" -lt 2 ]; then | |
| echo "Usage: ./newt rlaunch-exact 16d7 OPTIONAL_ROI_NAME" | |
| exit 1 | |
| elif [ "$#" -gt 3 ]; then | |
| echo "Usage: ./newt rlaunch-exact 16d7 OPTIONAL_ROI_NAME" | |
| exit 1 | |
| fi | |
| if [ -z "$2" ]; then | |
| echo "FATAL: Could not parse repo while generating configs" | |
| exit 1 | |
| else | |
| AUTOREPO=$2 | |
| fi | |
| # END ARGUMENT PARSING | |
| LAUNCH_MODE="exact" | |
| launch_guided_neutu | |
| elif [ "$1" == "rlaunch-dev" ]; then | |
| # This is just like rlaunch, but with the "dev" install of neutu | |
| # "dev" is just whatever version is current on flyem-forge, rather than our | |
| # curated version | |
| export USE_SEQUENCER=yes | |
| # BEGIN ARGUMENT PARSING | |
| if [ "$#" -eq 3 ]; then | |
| ROI=$3 | |
| elif [ "$#" -lt 2 ]; then | |
| echo "Usage: ./newt rlaunch-dev 16d7 OPTIONAL_ROI_NAME" | |
| exit 1 | |
| elif [ "$#" -gt 3 ]; then | |
| echo "Usage: ./newt rlaunch-dev 16d7 OPTIONAL_ROI_NAME" | |
| exit 1 | |
| fi | |
| if [ -z "$2" ]; then | |
| echo "FATAL: Could not parse repo while generating configs" | |
| exit 1 | |
| else | |
| AUTOREPO=$2 | |
| fi | |
| # END ARGUMENT PARSING | |
| LAUNCH_MODE="normal" | |
| launch_guided_devel_neutu | |
| elif [ "$1" == "rlaunch-debug" ]; then | |
| # This is just like rlaunch, but with the "debug" install of neutu, usually a | |
| # debug build that Ting gave us | |
| echo "Launching a debug version of neutu!!" | |
| export USE_SEQUENCER=yes | |
| # BEGIN ARGUMENT PARSING | |
| if [ "$#" -eq 3 ]; then | |
| ROI=$3 | |
| elif [ "$#" -lt 2 ]; then | |
| echo "Usage: ./newt rlaunch-debug 16d7 OPTIONAL_ROI_NAME" | |
| exit 1 | |
| elif [ "$#" -gt 3 ]; then | |
| echo "Usage: ./newt rlaunch-debug 16d7 OPTIONAL_ROI_NAME" | |
| exit 1 | |
| fi | |
| if [ -z "$2" ]; then | |
| echo "FATAL: Could not parse repo while generating configs" | |
| exit 1 | |
| else | |
| AUTOREPO=$2 | |
| fi | |
| # END ARGUMENT PARSING | |
| LAUNCH_MODE="normal" | |
| launch_guided_debug_neutu | |
| elif [ "$1" == "rlaunch-local" ]; then | |
| # This is intended for debugging alternative segmentation stacks running on the local | |
| # system. It is not compatible with ROIs (not for any particular reason, but I doubt we'll | |
| # ever care to do that kind of debugging) | |
| export USE_SEQUENCER=yes | |
| # BEGIN ARGUMENT PARSING | |
| if [ "$#" -ne 3 ]; then | |
| echo "Sample Usage: ./newt rlaunch-local 16d7 wasp1" | |
| exit 1 | |
| fi | |
| if [ -z "$2" ]; then | |
| echo "FATAL: Could not parse repo while generating configs" | |
| exit 1 | |
| else | |
| AUTOREPO=$2 | |
| fi | |
| if [ -z "$3" ]; then | |
| echo "FATAL: Could not parse specimen while generating configs" | |
| exit 1 | |
| else | |
| SPEC=$3 | |
| fi | |
| # END ARGUMENT PARSING | |
| LAUNCH_MODE="normal" | |
| launch_guided_local_neutu | |
| elif [ "$1" == "rlaunch-localish" ]; then | |
| # This is intended for debugging alternative segmentation stacks running on the local(ish) | |
| # system. Almost the same semantics as rlaunch-local but you also have a hostname in there | |
| export USE_SEQUENCER=yes | |
| # BEGIN ARGUMENT PARSING | |
| if [ "$#" -ne 4 ]; then | |
| echo "Sample Usage: ./newt rlaunch-localish hostname 16d7 wasp1" | |
| exit 1 | |
| fi | |
| if [ -z "$2" ]; then | |
| echo "FATAL: You must provide a hostname" | |
| exit 1 | |
| else | |
| SEG_HOST=$2 | |
| fi | |
| if [ -z "$3" ]; then | |
| echo "FATAL: Could not parse repo while generating configs" | |
| exit 1 | |
| else | |
| AUTOREPO=$3 | |
| fi | |
| if [ -z "$4" ]; then | |
| echo "FATAL: Could not parse specimen while generating configs" | |
| exit 1 | |
| else | |
| SPEC=$4 | |
| fi | |
| # END ARGUMENT PARSING | |
| LAUNCH_MODE="normal" | |
| launch_guided_localish_neutu | |
| elif [ "$1" == "launchnos" ]; then | |
| LAUNCH_MODE="normal" | |
| launch_neutu | |
| elif [ "$1" == "launch3" ]; then | |
| LAUNCH_MODE="normal" | |
| export USE_SEQUENCER=yes | |
| launch_neuthree | |
| elif [ "$1" == "rlaunch3" ]; then | |
| LAUNCH_MODE="normal" | |
| export USE_SEQUENCER=yes | |
| if [ -z "$2" ]; then | |
| echo "FATAL: Could not parse repo while generating configs" | |
| exit 1 | |
| else | |
| AUTOREPO=$2 | |
| fi | |
| launch_guided_neuthree | |
| elif [ "$1" == "view" ]; then | |
| if [ -z "$2" ]; then | |
| echo "FATAL: Could not parse specimenid while generating configs: use wasp1, wasp2, or wasp2a" | |
| exit 1 | |
| fi | |
| # $2 is specimen, like wasp1, wasp2... | |
| launch_viewmode_neutu $2 | |
| elif [ "$1" == "skel" ]; then | |
| # Launch neutu in command:skeletonization mode, to generate and upload skeletons to DVID | |
| if [ "$#" -ne 3 ]; then | |
| echo "Sample usage: ./newt skel 020cc:main 12501" | |
| exit 1 | |
| fi | |
| launch_the_skeletons $2 $3 | |
| elif [ "$1" == "devel" ]; then | |
| conda deactivate | |
| conda clean -y --all | |
| $MAMBA env remove -n $DEVEL_ENV || true # Allow this to fail - old environment might not exist | |
| $MAMBA create -y -n $DEVEL_ENV -c flyem-forge -c conda-forge neuclease "numpy>=1.26.0,<2.0.0" "pandas>=2.2.0" python | |
| echo "Neuclease development environment created" | |
| elif [ "$1" == "devel-jupyter" ]; then | |
| conda deactivate | |
| conda clean -y --all | |
| $MAMBA env remove -n $DEVEL_JUP_ENV || true # Allow this to fail - old environment might not exist | |
| $MAMBA create -y -n $DEVEL_JUP_ENV -c flyem-forge -c conda-forge neuclease "numpy>=1.26.0,<2.0.0" "pandas>=2.2.0" python jupyter | |
| echo "Neuclease development environment with jupyter created" | |
| elif [ "$1" == "update" ]; then | |
| conda install -y -n base -c conda-forge $NEWT_ALWAYS_INSTALL_BASE | |
| echo "Removing old environment" | |
| conda deactivate | |
| echo "I am cleaning your conda install. This may take a bit the first time" | |
| conda clean -y --all | |
| $MAMBA env remove -n $NEUTU_ENV || true # Allow this to fail - old environment might not exist | |
| echo "Creating new environment" | |
| # XXX Temporary reversion | |
| $MAMBA create -y -n $NEUTU_ENV -c flyem-forge -c conda-forge neutu-develop=$PIN_NEUTU neu3-develop=$PIN_NEUTHREE | |
| #$MAMBA create -y -n $NEUTU_ENV -c flyem-forge -c conda-forge neutu-develop=$PIN_NEUTU neu3-develop=$PIN_NEUTHREE libdvid-cpp=$PIN_LIBDVID_CPP | |
| echo "New environment created" | |
| elif [ "$1" == "update-dev" ]; then | |
| conda install -y -n base -c conda-forge $NEWT_ALWAYS_INSTALL_BASE | |
| echo "Removing old dev environment" | |
| conda deactivate | |
| echo "I am cleaning your conda install. This may take a bit the first time" | |
| conda clean -y --all | |
| $MAMBA env remove -n $NEUTU_DEV_ENV || true # Allow this to fail - old environment might not exist | |
| echo "Creating new dev env..." | |
| # This lone pin may or may not cause problems in the future, but some installs seem oddly insistent on delivering a version of libdvid-cpp | |
| # that as-of-this-writing does not work with any released version of neutu. This code will likely need to change in the future. | |
| $MAMBA create -y -n $NEUTU_DEV_ENV -c flyem-forge -c conda-forge neutu-develop neu3-develop | |
| echo "New dev environment created" | |
| elif [ "$1" == "update-debug" ]; then | |
| echo "This command not currently supported" | |
| exit 1 | |
| conda install -y -n base -c conda-forge $NEWT_ALWAYS_INSTALL_BASE | |
| echo "Removing old debug environment" | |
| conda deactivate | |
| echo "I am cleaning your conda install. This may take a bit the first time" | |
| conda clean -y --all | |
| $MAMBA env remove -n $NEUTU_DEBUG_ENV || true # Allow this to fail - old environment might not exist | |
| echo "Creating new debug env..." | |
| # This lone pin may or may not cause problems in the future, but some installs seem oddly insistent on delivering a version of libdvid-cpp | |
| # that as-of-this-writing does not work with any released version of neutu. This code will likely need to change in the future. | |
| $MAMBA create -y -n $NEUTU_DEBUG_ENV -c tingzhao -c flyem-forge -c conda-forge neutu-debug | |
| echo "New debug environment created!!" | |
| elif [ "$1" == "update-prereqs" ]; then | |
| conda install -y -n base -c conda-forge $NEWT_ALWAYS_INSTALL_BASE | |
| elif [ "$1" == "update-newt" ]; then | |
| echo "Updating newt" | |
| rm -f $NEWT_TEMPFILE | |
| $CURL $NEWT_GIST_URL -o $NEWT_TEMPFILE | |
| chmod 755 $NEWT_TEMPFILE | |
| echo "Performing self-update of newt manager" | |
| /bin/mv $NEWT_TEMPFILE $NEWT_PATH && exit 0 | |
| elif [ "$1" == "info" ]; then | |
| echo "NeuTu is version $(conda list -n managed_neutu neutu-develop | grep neutu-develop | tr -s ' ' | cut -d" " -f2)" | |
| echo "Latest NeuTu in flyem-forge is version $(conda search -c flyem-forge neutu-develop | tail -1 | tr -s ' ' | cut -d" " -f2)" | |
| if [ "$(uname)" == "Darwin" ]; then | |
| echo "Your system has $(hwprefs cpu_count) cores and $(hwprefs memory_size) memory" | |
| else # Assume Linux | |
| echo "Your system has $(lscpu -p -y | grep -v "^#" | wc -l) cores and $(free --giga | grep Mem | tr -s ' ' | cut -d" " -f2)G memory" | |
| fi | |
| #echo "Latest NeuTu in dev (not available via this tool) is version $(conda search -c tingzhao neutu-develop | tail -1 | tr -s ' ' | cut -d" " -f2)" | |
| elif [ "$1" == "config" ]; then | |
| resolve_config | |
| echo "You can use this subcommand to view the current config, or to change it" | |
| if [ "$#" -eq 1 ]; then | |
| echo "Config:" | |
| display_all_config_and_options | |
| echo "To set a value, do something like ./newt config WASP2_IMGSET tch1" | |
| else | |
| resolve_config | |
| set_config_item $2 $3 | |
| echo "I will parse further args to set config options" | |
| fi | |
| else | |
| echo "Usage: newt COMMAND" | |
| echo "Commands:" | |
| echo -e "\trlaunch REPO OPTIONAL_ROI -- Launch neutu against named repo (do not provide :main)" | |
| echo -e "\trlaunch-exact REPO OPTIONAL_ROI -- Launch neutu against exact UUID (mostly intended for historical view)" | |
| echo -e "\tlaunch3 -- Launch neu3" | |
| echo -e "\trlaunch3 REPO -- Launch neu3 against named repo (do not provide :main)" | |
| echo -e "\tskel REPO BODYSPEC -- Generate and upload skeletons for either a provided numeric body, or for all bodies found in a textfile (that must end in .txt)" | |
| echo -e "\tupdate -- Update/install neutu" | |
| echo -e "\tupdate-newt -- Update the newt wrapper itself" | |
| echo -e "\tdevel -- Set up a neuclease conda environment" | |
| echo -e "\tdevel-jupyter -- Set up a neuclease conda environment with Jupyter" | |
| echo -e "\tinfo -- Show info on your system" | |
| fi | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment