Skip to content

Instantly share code, notes, and snippets.

@jasonrhaas

jasonrhaas/config.json

Last active August 29, 2015 14:14
Show Gist options
  • Save jasonrhaas/e8fa64b7ebeb3666c21c to your computer and use it in GitHub Desktop.
Save jasonrhaas/e8fa64b7ebeb3666c21c to your computer and use it in GitHub Desktop.
Download ZIP
Streamparse deploy errors
Raw
config.json
{
"library": "",
"topology_specs": "topologies/",
"virtualenv_specs": "virtualenvs/",
"envs": {
"prod": {
"user": "",
"nimbus": "",
"workers": [],
"log": {
"path": "",
"max_bytes": 1000000,
"backup_count": 10,
"level": "info"
},
"virtualenv_root": ""
},
"memex": {
"user": "storm",
"nimbus": "memex-storm-master.xdata.data-tactics-corp.com",
"workers": ["r102u25.xdata.data-tactics-corp.com",
"r102u26.xdata.data-tactics-corp.com",
"r102u27.xdata.data-tactics-corp.com",
"r102u28.xdata.data-tactics-corp.com",
"r102u29.xdata.data-tactics-corp.com",
"r102u30.xdata.data-tactics-corp.com",
"r102u31.xdata.data-tactics-corp.com",
"r102u32.xdata.data-tactics-corp.com",
"r102u33.xdata.data-tactics-corp.com",
"r103u20.xdata.data-tactics-corp.com",
"r103u21.xdata.data-tactics-corp.com",
"r103u22.xdata.data-tactics-corp.com",
"r103u23.xdata.data-tactics-corp.com",
"r103u24.xdata.data-tactics-corp.com",
"r103u25.xdata.data-tactics-corp.com",
"r103u26.xdata.data-tactics-corp.com",
"r103u27.xdata.data-tactics-corp.com",
"r103u28.xdata.data-tactics-corp.com",
"r103u29.xdata.data-tactics-corp.com",
"r103u30.xdata.data-tactics-corp.com",
"r103u31.xdata.data-tactics-corp.com"],
"log": {
"path": "/var/log/storm",
"max_bytes": 1000000,
"backup_count": 10,
"level": "info"
},
"provisioner": "conda",
"conda": {
"python_version": "2.7",
"conda_root": "/srv/software/anaconda",
"conda_env_root": "/srv/software/storm/envs"
},
"virtualenv": {
"virtualenv_root": "/srv/software/streamparse/envs"
},
"shell": {
"remote_script": "setup.sh"
}
},
"ist": {
"user": "storm",
"nimbus": "ist-storm-master.istresearch.com",
"workers": ["ist-storm1.istresearch.com",
"ist-storm2.istresearch.com"],
"log": {
"path": "/tmp/apache-storm-0.9.2-incubating/logs",
"max_bytes": 1000000,
"backup_count": 10,
"level": "info"
},
"provisioner": "conda",
"conda": {
"python_version": "2.7",
"conda_root": "/tmp/miniconda",
"conda_env_root": "/tmp/conda_env_root"
},
"virtualenv": {
"virtualenv_root": "/tmp/virtualenv_root"
},
"shell": {
"remote_script": "setup.sh"
}
}
}
}
Raw
errors
emex-dev:~/repos/memex/streamparse/ist-extract$ sparse submit -e ist -v
[memex-sp01] Executing task '_create_or_update_virtualenv'
[memex-sp02] Executing task '_create_or_update_virtualenv'
!!! Parallel execution exception under host u'memex-sp02':
!!! Parallel execution exception under host u'memex-sp01':
Process memex-sp01:
Traceback (most recent call last):
File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python2.7/dist-packages/fabric/tasks.py", line 239, in inner
submit(task.run(*args, **kwargs))
File "/usr/local/lib/python2.7/dist-packages/fabric/tasks.py", line 174, in run
return self.wrapped(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/fabric/decorators.py", line 181, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/streamparse/ext/fabric.py", line 107, in _create_or_update_virtualenv
virtualenv_path = os.path.join(virtualenv_root, virtualenv_name)
File "/usr/lib/python2.7/posixpath.py", line 77, in join
elif path == '' or path.endswith('/'):
AttributeError: 'NoneType' object has no attribute 'endswith'
Process memex-sp02:
Traceback (most recent call last):
File "/usr/lib/python2.7/multiprocessing/process.py", line 258, in _bootstrap
self.run()
File "/usr/lib/python2.7/multiprocessing/process.py", line 114, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python2.7/dist-packages/fabric/tasks.py", line 239, in inner
submit(task.run(*args, **kwargs))
File "/usr/local/lib/python2.7/dist-packages/fabric/tasks.py", line 174, in run
return self.wrapped(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/fabric/decorators.py", line 181, in inner
return func(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/streamparse/ext/fabric.py", line 107, in _create_or_update_virtualenv
virtualenv_path = os.path.join(virtualenv_root, virtualenv_name)
File "/usr/lib/python2.7/posixpath.py", line 77, in join
elif path == '' or path.endswith('/'):
AttributeError: 'NoneType' object has no attribute 'endswith'
Fatal error: One or more hosts failed while executing task '_create_or_update_virtualenv'
Underlying exception:
'NoneType' object has no attribute 'endswith'
Aborting.
One or more hosts failed while executing task '_create_or_update_virtualenv'
Underlying exception:
'NoneType' object has no attribute 'endswith'
Raw
new_config.json
{
"library": "",
"topology_specs": "topologies/",
"virtualenv_specs": "virtualenvs/",
"envs": {
"prod": {
"user": "",
"nimbus": "",
"workers": [],
"log": {
"path": "",
"max_bytes": 1000000,
"backup_count": 10,
"level": "info"
},
"virtualenv_root": ""
},
"memex": {
"user": "storm",
"nimbus": "memex-storm-master.xdata.data-tactics-corp.com",
"workers": ["r102u25.xdata.data-tactics-corp.com",
"r102u26.xdata.data-tactics-corp.com",
"r102u27.xdata.data-tactics-corp.com",
"r102u28.xdata.data-tactics-corp.com",
"r102u29.xdata.data-tactics-corp.com",
"r102u30.xdata.data-tactics-corp.com",
"r102u31.xdata.data-tactics-corp.com",
"r102u32.xdata.data-tactics-corp.com",
"r102u33.xdata.data-tactics-corp.com",
"r103u20.xdata.data-tactics-corp.com",
"r103u21.xdata.data-tactics-corp.com",
"r103u22.xdata.data-tactics-corp.com",
"r103u23.xdata.data-tactics-corp.com",
"r103u24.xdata.data-tactics-corp.com",
"r103u25.xdata.data-tactics-corp.com",
"r103u26.xdata.data-tactics-corp.com",
"r103u27.xdata.data-tactics-corp.com",
"r103u28.xdata.data-tactics-corp.com",
"r103u29.xdata.data-tactics-corp.com",
"r103u30.xdata.data-tactics-corp.com",
"r103u31.xdata.data-tactics-corp.com"],
"log": {
"path": "/var/log/storm",
"max_bytes": 1000000,
"backup_count": 10,
"level": "info"
},
"provisioner": "conda",
"conda": {
"python_version": "2.7",
"conda_root": "/srv/software/anaconda",
"conda_env_root": "/srv/software/storm/envs"
},
"virtualenv": {
"virtualenv_root": "/srv/software/streamparse/envs"
},
"shell": {
"remote_script": "setup.sh"
}
},
"ist": {
"user": "storm",
"nimbus": "ist-storm-master.istresearch.com",
"workers": ["ist-storm1.istresearch.com",
"ist-storm2.istresearch.com"],
"log": {
"path": "/tmp/apache-storm-0.9.2-incubating/logs",
"max_bytes": 1000000,
"backup_count": 10,
"level": "info"
},
"virtualenv_root": "/tmp/virtualenv_root"
}
}
}
Raw
new_config2.json
{
"library": "",
"topology_specs": "topologies/",
"virtualenv_specs": "virtualenvs/",
"envs": {
"prod": {
"user": "",
"nimbus": "",
"workers": [],
"log": {
"path": "",
"max_bytes": 1000000,
"backup_count": 10,
"level": "info"
},
"virtualenv_root": ""
},
"memex": {
"user": "storm",
"nimbus": "memex-storm-master.xdata.data-tactics-corp.com",
"workers": ["r102u25.xdata.data-tactics-corp.com",
"r102u26.xdata.data-tactics-corp.com",
"r102u27.xdata.data-tactics-corp.com",
"r102u28.xdata.data-tactics-corp.com",
"r102u29.xdata.data-tactics-corp.com",
"r102u30.xdata.data-tactics-corp.com",
"r102u31.xdata.data-tactics-corp.com",
"r102u32.xdata.data-tactics-corp.com",
"r102u33.xdata.data-tactics-corp.com",
"r103u20.xdata.data-tactics-corp.com",
"r103u21.xdata.data-tactics-corp.com",
"r103u22.xdata.data-tactics-corp.com",
"r103u23.xdata.data-tactics-corp.com",
"r103u24.xdata.data-tactics-corp.com",
"r103u25.xdata.data-tactics-corp.com",
"r103u26.xdata.data-tactics-corp.com",
"r103u27.xdata.data-tactics-corp.com",
"r103u28.xdata.data-tactics-corp.com",
"r103u29.xdata.data-tactics-corp.com",
"r103u30.xdata.data-tactics-corp.com",
"r103u31.xdata.data-tactics-corp.com"],
"log": {
"path": "/var/log/storm",
"max_bytes": 1000000,
"backup_count": 10,
"level": "info"
},
"provisioner": "conda",
"conda": {
"python_version": "2.7",
"conda_root": "/srv/software/anaconda",
"conda_env_root": "/srv/software/storm/envs"
},
"virtualenv": {
"virtualenv_root": "/srv/software/streamparse/envs"
},
"shell": {
"remote_script": "setup.sh"
}
},
"ist": {
"user": "root",
"nimbus": "memex-nimbus",
"workers": ["memex-sp01",
"memex-sp02"],
"log": {
"path": "/tmp/apache-storm-0.9.2-incubating/logs",
"max_bytes": 1000000,
"backup_count": 10,
"level": "info"
},
"virtualenv_root": "/tmp/virtualenv_root"
}
}
}
Raw
project.clj
(defproject ist-extract "0.0.1-SNAPSHOT"
:source-paths ["src/clj" "topologies"]
:resource-paths ["_resources"]
:target-path "_build"
:min-lein-version "2.0.0"
:jvm-opts ["-client"]
:dependencies [[org.apache.storm/storm-core "0.9.2-incubating"]
[org.apache.storm/storm-kafka "0.9.2-incubating" :exclusions [org.slf4j/slf4j-api]]
[org.apache.kafka/kafka_2.9.2 "0.8.1.1" :exclusions [com.sun.jmx/jmxri com.sun.jdmk/jmxtools javax.jms/jms org.slf4j/slf4j-api]]
[org.apache.zookeeper/zookeeper "3.4.6" :exclusions [io.netty/netty org.slf4j/slf4j-api org.slf4j/slf4j-log4j12]]
[com.parsely/streamparse "0.0.4-SNAPSHOT"]
]
:jar-exclusions [#"log4j\.properties" #"backtype" #"trident" #"META-INF" #"meta-inf" #"\.yaml"]
:uberjar-exclusions [#"log4j\.properties" #"backtype" #"trident" #"META-INF" #"meta-inf" #"\.yaml"]
)
Raw
submit errors
clojure.main$null_opt.invoke (main.clj:362)
clojure.main$main.doInvoke (main.clj:440)
clojure.lang.RestFn.invoke (RestFn.java:421)
Exception in thread "main" java.lang.NullP clojure.lang.Var.invoke (Var.java:419)
ointerException
at streamparse.commands.submit_to clojure.lang.AFn.applyToHelper (AFn.java:163)
pology$submit_topology_BANG_.invoke(submit_t clojure.lang.Var.applyTo (Var.java:532)
opology.clj:27)
at streamparse.comma clojure.main.main (main.java:37)
nds.submit_topology$_main.doInvoke(submit_topology.clj:74)
at clojure.lang.RestFn.invoke(RestFn.java:2088)
at clojure.lang.Var.invoke(Var.java:489)
at user$eval5.invoke(form-init864970177701291312.clj:1)
at clojure.lang.Compiler.eval(Compiler.java:6619)
at clojure.lang.Compiler.eval(Compiler.java:6609)
at clojure.lang.Compiler.load(Compiler.java:7064)
at clojure.lang.Compiler.loadFile(Compiler.java:7020)
at clojure.main$load_script.invoke(main.clj:294)
at clojure.main$init_opt.invoke(main.clj:299)
at clojure.main$initialize.invoke(main.clj:327)
at clojure.main$null_opt.invoke(main.clj:362)
at clojure.main$main.doInvoke(main.clj:440)
at clojure.lang.RestFn.invoke(RestFn.java:421)
at clojure.lang.Var.invoke(Var.java:419)
at clojure.lang.AFn.applyToHelper(AFn.java:163)
at clojure.lang.Var.applyTo(Var.java:532)
at clojure.main.main(main.java:37)
Traceback (most recent call last):
File "/usr/local/bin/sparse", line 9, in <module>
load_entry_point('streamparse==1.1.0', 'console_scripts', 'sparse')()
File "/usr/local/lib/python2.7/dist-packages/streamparse/cmdln.py", line 94, in main
args["--force"], args["--debug"], args["--wait"])
File "/usr/local/lib/python2.7/dist-packages/invoke/tasks.py", line 111, in __call__
result = self.body(*args, **kwargs)
File "/usr/local/lib/python2.7/dist-packages/streamparse/ext/invoke.py", line 277, in submit_topology
run(full_cmd)
File "/usr/local/lib/python2.7/dist-packages/invoke/runner.py", line 160, in run
raise Failure(result)
invoke.exceptions.Failure: Command execution failure!
Exit code: 1
Stderr:
Exception in thread "main" java.lang.NullPointerException
at streamparse.commands.submit_topology$submit_topology_BANG_.invoke(submit_topology.clj:27)
at streamparse.commands.submit_topology$_main.doInvoke(submit_topology.clj:74)
at clojure.lang.RestFn.invoke(RestFn.java:2088)
at clojure.lang.Var.invoke(Var.java:489)
at user$eval5.invoke(form-init864970177701291312.clj:1)
at clojure.lang.Compiler.eval(Compiler.java:6619)
at clojure.lang.Compiler.eval(Compiler.java:6609)
at clojure.lang.Compiler.load(Compiler.java:7064)
at clojure.lang.Compiler.loadFile(Compiler.java:7020)
at clojure.main$load_script.invoke(main.clj:294)
at clojure.main$init_opt.invoke(main.clj:299)
at clojure.main$initialize.invoke(main.clj:327)
at clojure.main$null_opt.invoke(main.clj:362)
at clojure.main$main.doInvoke(main.clj:440)
at clojure.lang.RestFn.invoke(RestFn.java:421)
at clojure.lang.Var.invoke(Var.java:419)
at clojure.lang.AFn.applyToHelper(AFn.java:163)
at clojure.lang.Var.applyTo(Var.java:532)
at clojure.main.main(main.java:37)
Raw
virtualenv
[root@ip-10-12-17-223 bin]# ./pip install virtualenv
WARNING: using virtualenv with Anaconda is untested and not recommended.
We suggest using the conda command to create environments instead.
For more information about creating conda environments, please see:
http://docs.continuum.io/conda/examples/create.html
Proceed (y/n)? n
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment