twobraids

#application='ProcessorApp'

[resource]

    [[fs]]

        # this section contains Options that are common in
        # other sections of this ini file.  If they are used
        # in other files too, copy the options to the file
        # in the +include line below.  Comment out the values

twobraids

	

    I once saw some really opaque code;
    while thinking this is just a big load;
    ought to write it again;
    to do less is a sin;
    "choose writing over reading", I crowed

twobraids

# name: application
# doc: the name of the app to run (select from: collector, copy_processed, copy_raw_and_processed, crashmover, fetch, middleware, processor, reprocess_crashlist, setupdb, submitter)
application='socorro.collector.crashmover_app.RawAndProcessedCopierApp'

#-------------------------------------------------------------------------------
# source - 

# name: source.crashstorage_class
# doc: the source storage class
source.crashstorage_class='socorro.external.boto.crashstorage.BotoS3CrashStorage'

twobraids

from configman.dotdict import DotDict
from socorro.processor.signature_utilities import CSignatureToolBase

c = DotDict()
c.collapse_arguments = True
ct = CSignatureToolBase(c)

signature_with_arguments = "@0xb6f11e3c | some_func() | aont(onth()) | operator()(ortho)"

signature_with_no_arguments = ct._collapse(

twobraids

socorroᛤ ./scripts/sigtool.py --help
usage:
./scripts/sigtool.py [OPTIONS]...
[ signature ] 

OPTIONS:
  --admin.conf
    the pathname of the config file (path/filename)

  --admin.dump_conf

twobraids

[centos@i-a289086a ~]$ sudo grep "saved" /var/log/messages | wc -l
1000
[centos@i-a289086a ~]$ sudo grep "RabbitMQCrashStorage saving" /var/log/messages | wc -l
664

[centos@i-6a8607a2 ~]$ sudo grep acking /var/log/messages | grep processor | grep i-6a8607a2  | wc -l
666
[centos@i-6a8607a2 ~]$ sudo grep starting /var/log/messages | grep processor | grep i-6a8607a2  | wc -l
666

twobraids

Jul 17 13:31:42 i-29100bde bash: 2015-07-17 13:31:42,148 DEBUG - processor -  - Thread-3 - BotoBenchmarkRead get_raw_crash 0:00:00.161613
Jul 17 13:31:42 i-29100bde bash: 2015-07-17 13:31:42,174 DEBUG - processor -  - Thread-3 - BotoBenchmarkRead get_raw_dumps_as_files 0:00:00.025596
Jul 17 13:31:42 i-29100bde bash: 2015-07-17 13:31:42,220 DEBUG - processor -  - Thread-3 - BotoBenchmarkRead get_unredacted_processed 0:00:00.045540
Jul 17 13:31:42 i-29100bde bash: 2015-07-17 13:31:42,220 INFO - processor -  - Thread-3 - starting job: 3ff56127-c0cf-48fd-bb7c-5d39d2150717
Jul 17 13:31:42 i-29100bde bash: 2015-07-17 13:31:42,221 DEBUG - processor -  - Thread-3 - Rule socorro.processor.general_transform_rules.CPUInfoRule action failed because of missing key "'json_dump'"
Jul 17 13:31:42 i-29100bde bash: 2015-07-17 13:31:42,221 DEBUG - processor -  - Thread-3 - Rule socorro.processor.general_transform_rules.OSInfoRule action failed because of missing key "'json_dump'"
Jul 17 13:31:42 i-29100bde bash: 2015-07-17 13:3

twobraids

[centos@i-46aa2a8e ~]$ sudo grep processor.raw_to_processed_transform.chatty_rules /var/log/messages 
Jul 17 14:24:07 ip-172-31-44-44 bash: 2015-07-17 14:24:07,479 INFO - processor -  - MainThread - processor.raw_to_processed_transform.chatty_rules: False
Jul 17 14:24:07 i-46aa2a8e bash: 2015-07-17 14:24:07,479 INFO - processor -  - MainThread - processor.raw_to_processed_transform.chatty_rules: False
[centos@i-46aa2a8e ~]$ consulate kv get processor.raw_to_processed_transform.chatty_rules
True
[centos@i-46aa2a8e ~]$ 

twobraids

2015-07-20 19:22:05,523 INFO - processor -  - Thread-13 - starting job: 2732e133-cc5a-47bd-a77e-cd84b2150720
2015-07-20 19:22:06,751 INFO - processor -  - QueuingThread - duplicate job: 2732e133-cc5a-47bd-a77e-cd84b2150720 is already in progress
2015-07-20 19:22:06,751 DEBUG - processor -  - QueuingThread - RabbitMQCrashStorage acking 2732e133-cc5a-47bd-a77e-cd84b2150720 with delivery_tag 845
2015-07-20 19:22:08,474 INFO - processor -  - Thread-13 - finishing successful job: 2732e133-cc5a-47bd-a77e-cd84b2150720
2015-07-20 19:22:08,802 INFO - processor -  - QueuingThread - duplicate job: 2732e133-cc5a-47bd-a77e-cd84b2150720 is already in progress
2015-07-20 19:22:08,802 DEBUG - processor -  - QueuingThread - RabbitMQCrashStorage acking 2732e133-cc5a-47bd-a77e-cd84b2150720 with delivery_tag 846
2015-07-20 19:22:15,863 INFO - processor -  - QueuingThread - duplicate job: 2732e133-cc5a-47bd-a77e-cd84b2150720 is already in progress
2015-07-20 19:22:15,864 DEBUG - processor -  - QueuingThread - RabbitMQCrashStorage

twobraids

# for prod crashmover

destination.storage_classes="socorro.external.rabbitmq.crashstorage.RabbitMQCrashStorage, socorro.external.boto.crashstorage.BotoS3CrashStorage, socorro.external.statsd.statsd_base.StatsdCounter, socorro.external.rabbitmq.crashstorage.RabbitMQCrashStorage"

destination.storage3.crashstorage_class=socorro.external.statsd.statsd_base.StatsdBenchmarkingWrapper
destination.storage3.filter_on_legacy_processing=False
destination.storage3.routing_key=socorro.stagesubmitter
destination.storage3.statsd_prefix=crashmover.rabbitmq_stagesubmitter
destination.storage3.active_list=transaction
destination.storage3.wrapped_object_class=socorro.external.rabbitmq.crashstorage.RabbitMQCrashStorage