fleetctl journal --lines 300 deis-database

gistfile1.txt

-- Logs begin at Wed 2014-11-05 00:21:35 UTC, end at Wed 2014-11-05 01:26:50 UTC. --
Nov 05 01:01:41 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal systemd[1]: Starting deis-database...
Nov 05 01:01:56 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[2379]: 2014/11/05 01:01:56 Error response from daemon: No such id: deis/database:v0.15.1
Nov 05 01:01:56 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[2379]: Pulling repository deis/database
Nov 05 01:18:30 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[2379]: Status: Downloaded newer image for deis/database:v0.15.1
Nov 05 01:18:31 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal systemd[1]: Started deis-database.
Nov 05 01:19:02 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: 2014-11-05T01:19:02Z 9d8486cd1565 confd[57]: ERROR exit status 127
Nov 05 01:19:02 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: 2014-11-05T01:19:02Z 9d8486cd1565 confd[57]: ERROR exit status 127
Nov 05 01:19:12 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: database: waiting for confd to write initial templates...
Nov 05 01:19:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: The files belonging to this database system will be owned by user "postgres".
Nov 05 01:19:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: This user must also own the server process.
Nov 05 01:19:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: The database cluster will be initialized with locale "C".
Nov 05 01:19:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: The default database encoding has accordingly been set to "SQL_ASCII".
Nov 05 01:19:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: The default text search configuration will be set to "english".
Nov 05 01:19:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: Data page checksums are disabled.
Nov 05 01:19:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: creating directory /var/lib/postgresql/9.3/main ... ok
Nov 05 01:19:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: creating subdirectories ... ok
Nov 05 01:19:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: selecting default max_connections ... 100
Nov 05 01:19:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: selecting default shared_buffers ... 128MB
Nov 05 01:19:22 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: creating configuration files ... ok
Nov 05 01:19:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: creating template1 database in /var/lib/postgresql/9.3/main/base/1 ... ok
Nov 05 01:19:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: initializing pg_authid ... ok
Nov 05 01:19:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: initializing dependencies ... ok
Nov 05 01:19:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: creating system views ... ok
Nov 05 01:19:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: loading system objects' descriptions ... ok
Nov 05 01:19:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: creating collations ... ok
Nov 05 01:19:28 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: creating conversions ... ok
Nov 05 01:19:28 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: creating dictionaries ... ok
Nov 05 01:19:28 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: setting privileges on built-in objects ... ok
Nov 05 01:19:32 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: creating information schema ... ok
Nov 05 01:19:32 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: loading PL/pgSQL server-side language ... ok
Nov 05 01:19:32 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: vacuuming database template1 ... ok
Nov 05 01:19:32 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: copying template1 to template0 ... ok
Nov 05 01:19:32 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: copying template1 to postgres ... ok
Nov 05 01:19:40 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: syncing data to disk ... ok
Nov 05 01:19:40 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: WARNING: enabling "trust" authentication for local connections
Nov 05 01:19:40 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: You can change this by editing pg_hba.conf or using the option -A, or
Nov 05 01:19:40 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: --auth-local and --auth-host, the next time you run initdb.
Nov 05 01:19:40 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: Success. You can now start the database server using:
Nov 05 01:19:40 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: /usr/lib/postgresql/9.3/bin/postgres -D /var/lib/postgresql/9.3/main
Nov 05 01:19:40 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: or
Nov 05 01:19:40 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: /usr/lib/postgresql/9.3/bin/pg_ctl -D /var/lib/postgresql/9.3/main -l logfile start
Nov 05 01:20:05 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: database: no existing database found.
Nov 05 01:20:09 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: database: no backups found. Initializing a new database...
Nov 05 01:20:11 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: psql: could not connect to server: No such file or directory
Nov 05 01:20:11 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: Is the server running locally and accepting
Nov 05 01:20:11 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: connections on Unix domain socket "/var/run/postgresql/.s.PGSQL.5432"?
Nov 05 01:20:11 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: database: performing an initial backup...
Nov 05 01:20:11 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: wal_e.main   INFO     MSG: starting WAL-E
Nov 05 01:20:11 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: DETAIL: The subcommand is "backup-push".
Nov 05 01:20:11 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: STRUCTURED: time=2014-11-05T01:20:11.117621-00 pid=181
Nov 05 01:20:13 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: psql: could not connect to server: No such file or directory
Nov 05 01:20:13 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: Is the server running locally and accepting
Nov 05 01:20:13 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: connections on Unix domain socket "/var/run/postgresql/.s.PGSQL.5432"?
Nov 05 01:20:13 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: wal_e.main   ERROR    MSG: could not run one or more external programs WAL-E depends upon
Nov 05 01:20:13 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: DETAIL: Could not get a connection to the database: note that superuser access
Nov 05 01:20:13 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: is required
Nov 05 01:20:13 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: Could not run the following programs, are they installed? psql
Nov 05 01:20:13 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[4948]: STRUCTURED: time=2014-11-05T01:20:13.249199-00 pid=181
Nov 05 01:20:13 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal systemd[1]: deis-database.service: main process exited, code=exited, status=1/FAILURE
Nov 05 01:20:13 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal bash[5891]: nsenter: neither filename nor target pid supplied for ns/ipc
Nov 05 01:20:14 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal docker[5905]: deis-database
Nov 05 01:20:14 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal systemd[1]: Unit deis-database.service entered failed state.
Nov 05 01:20:20 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal systemd[1]: deis-database.service holdoff time over, scheduling restart.
Nov 05 01:20:20 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal systemd[1]: Stopping deis-database...
Nov 05 01:20:20 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal systemd[1]: Starting deis-database...
Nov 05 01:20:20 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal systemd[1]: Started deis-database.
Nov 05 01:20:52 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05T01:20:52Z c32ffa9cad64 confd[54]: ERROR exit status 127
Nov 05 01:20:52 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05T01:20:52Z c32ffa9cad64 confd[54]: ERROR exit status 127
Nov 05 01:20:55 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: database: waiting for confd to write initial templates...
Nov 05 01:21:15 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: database: existing data directory found. Starting postgres...
Nov 05 01:21:16 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:21:16 UTC LOG:  database system was shut down at 2014-11-05 01:19:32 UTC
Nov 05 01:21:16 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:21:16 UTC FATAL:  the database system is starting up
Nov 05 01:21:16 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: psql: FATAL:  the database system is starting up
Nov 05 01:21:16 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: database: postgres is running...
Nov 05 01:21:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:21:19 UTC LOG:  autovacuum launcher started
Nov 05 01:21:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:21:19 UTC LOG:  database system is ready to accept connections
Nov 05 01:21:37 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:21:37 UTC FATAL:  password authentication failed for user "deis"
Nov 05 01:21:37 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:21:37 UTC DETAIL:  Connection matched pg_hba.conf line 102: "host         all                         all                         0.0.0.0/0                                 md5"
Nov 05 01:22:22 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: wal_e.main   INFO     MSG: starting WAL-E
Nov 05 01:22:22 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: DETAIL: The subcommand is "wal-push".
Nov 05 01:22:22 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: STRUCTURED: time=2014-11-05T01:22:22.879039-00 pid=296
Nov 05 01:22:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: wal_e.worker.upload INFO     MSG: begin archiving a file
Nov 05 01:22:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: DETAIL: Uploading "pg_xlog/000000010000000000000001" to "s3://db_wal/wal_005/000000010000000000000001.lzo".
Nov 05 01:22:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: STRUCTURED: time=2014-11-05T01:22:23.704725-00 pid=296 action=push-wal key=s3://db_wal/wal_005/000000010000000000000001.lzo prefix= seg=000000010000000000000001 state=begin
Nov 05 01:22:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:22:25 UTC FATAL:  password authentication failed for user "deis"
Nov 05 01:22:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:22:25 UTC DETAIL:  Connection matched pg_hba.conf line 102: "host         all                         all                         0.0.0.0/0                                 md5"
Nov 05 01:23:19 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05T01:23:19Z c32ffa9cad64 confd[108]: ERROR Config check failed: exit status 1
Nov 05 01:23:44 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05T01:23:44Z c32ffa9cad64 confd[108]: ERROR Config check failed: exit status 1
Nov 05 01:23:49 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05T01:23:49Z c32ffa9cad64 confd[108]: ERROR Config check failed: exit status 1
Nov 05 01:24:00 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05T01:24:00Z c32ffa9cad64 confd[108]: ERROR Config check failed: exit status 1
Nov 05 01:24:15 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05T01:24:15Z c32ffa9cad64 confd[108]: ERROR Config check failed: exit status 1
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: Traceback (most recent call last):
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/greenlet.py", line 327, in run
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: result = self._run(*self.args, **self.kwargs)
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/worker/upload.py", line 43, in __call__
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: self.gpg_key_id)
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/worker/worker_util.py", line 40, in do_lzop_put
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: k = blobstore.uri_put_file(creds, url, tf)
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/blobstore/s3/s3_util.py", line 54, in uri_put_file
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: k.set_contents_from_file(fp, encrypt_key=True)
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/s3/key.py", line 1291, in set_contents_from_file
Nov 05 01:24:21 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: chunked_transfer=chunked_transfer, size=size)
Nov 05 01:24:22 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/s3/key.py", line 748, in send_file
Nov 05 01:24:22 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: chunked_transfer=chunked_transfer, size=size)
Nov 05 01:24:22 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/s3/key.py", line 949, in _send_file_internal
Nov 05 01:24:22 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: query_args=query_args
Nov 05 01:24:22 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/s3/connection.py", line 664, in make_request
Nov 05 01:24:22 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: retry_handler=retry_handler
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/connection.py", line 1068, in make_request
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: retry_handler=retry_handler)
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/connection.py", line 939, in _mexe
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: request.body, request.headers)
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/s3/key.py", line 877, in sender
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: response = http_conn.getresponse()
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/cmd.py", line 32, in monkey
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: return original(*args, **kwargs)
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/lib/python2.7/httplib.py", line 1045, in getresponse
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: response.begin()
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/lib/python2.7/httplib.py", line 409, in begin
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: version, status, reason = self._read_status()
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/lib/python2.7/httplib.py", line 365, in _read_status
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: line = self.fp.readline(_MAXLINE + 1)
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/lib/python2.7/socket.py", line 476, in readline
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: data = self._sock.recv(self._rbufsize)
Nov 05 01:24:23 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/socket.py", line 392, in recv
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: self._wait(self._read_event)
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/socket.py", line 298, in _wait
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: self.hub.wait(watcher)
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 341, in wait
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: result = waiter.get()
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 568, in get
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: return self.hub.switch()
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 331, in switch
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: return greenlet.switch(self)
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: timeout: timed out
Nov 05 01:24:24 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: <Greenlet at 0x7fe66a55fd70: <wal_e.worker.upload.WalUploader object at 0x7fe66a2b6750>(<wal_e.worker.pg.wal_transfer.WalSegment object at)> failed with timeout
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: wal_e.main   CRITICAL MSG: An unprocessed exception has avoided all error handling
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: DETAIL: Traceback (most recent call last):
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/cmd.py", line 585, in main
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: concurrency=args.pool_size)
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/operator/backup.py", line 282, in wal_archive
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: group.join()
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/worker/pg/wal_transfer.py", line 148, in join
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: raise val
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: timeout: timed out
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: STRUCTURED: time=2014-11-05T01:24:25.831665-00 pid=296
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:24:25 UTC LOG:  archive command failed with exit code 2
Nov 05 01:24:25 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:24:25 UTC DETAIL:  The failed archive command was: envdir /etc/wal-e.d/env wal-e wal-push pg_xlog/000000010000000000000001
Nov 05 01:24:26 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: wal_e.main   INFO     MSG: starting WAL-E
Nov 05 01:24:26 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: DETAIL: The subcommand is "wal-push".
Nov 05 01:24:26 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: STRUCTURED: time=2014-11-05T01:24:26.949349-00 pid=494
Nov 05 01:24:26 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: wal_e.worker.upload INFO     MSG: begin archiving a file
Nov 05 01:24:26 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: DETAIL: Uploading "pg_xlog/000000010000000000000001" to "s3://db_wal/wal_005/000000010000000000000001.lzo".
Nov 05 01:24:26 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: STRUCTURED: time=2014-11-05T01:24:26.963615-00 pid=494 action=push-wal key=s3://db_wal/wal_005/000000010000000000000001.lzo prefix= seg=000000010000000000000001 state=begin
Nov 05 01:24:32 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:24:32 UTC FATAL:  password authentication failed for user "deis"
Nov 05 01:24:32 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:24:32 UTC DETAIL:  Connection matched pg_hba.conf line 102: "host         all                         all                         0.0.0.0/0                                 md5"
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: Traceback (most recent call last):
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/greenlet.py", line 327, in run
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: result = self._run(*self.args, **self.kwargs)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/worker/upload.py", line 43, in __call__
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: self.gpg_key_id)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/worker/worker_util.py", line 40, in do_lzop_put
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: k = blobstore.uri_put_file(creds, url, tf)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/blobstore/s3/s3_util.py", line 54, in uri_put_file
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: k.set_contents_from_file(fp, encrypt_key=True)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/s3/key.py", line 1291, in set_contents_from_file
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: chunked_transfer=chunked_transfer, size=size)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/s3/key.py", line 748, in send_file
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: chunked_transfer=chunked_transfer, size=size)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/s3/key.py", line 949, in _send_file_internal
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: query_args=query_args
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/s3/connection.py", line 664, in make_request
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: retry_handler=retry_handler
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/connection.py", line 1068, in make_request
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: retry_handler=retry_handler)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/connection.py", line 939, in _mexe
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: request.body, request.headers)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/boto/s3/key.py", line 877, in sender
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: response = http_conn.getresponse()
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/cmd.py", line 32, in monkey
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: return original(*args, **kwargs)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/lib/python2.7/httplib.py", line 1045, in getresponse
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: response.begin()
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/lib/python2.7/httplib.py", line 409, in begin
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: version, status, reason = self._read_status()
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/lib/python2.7/httplib.py", line 365, in _read_status
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: line = self.fp.readline(_MAXLINE + 1)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/lib/python2.7/socket.py", line 476, in readline
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: data = self._sock.recv(self._rbufsize)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/socket.py", line 392, in recv
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: self._wait(self._read_event)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/socket.py", line 298, in _wait
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: self.hub.wait(watcher)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 341, in wait
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: result = waiter.get()
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 568, in get
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: return self.hub.switch()
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/gevent/hub.py", line 331, in switch
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: return greenlet.switch(self)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: timeout: timed out
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: <Greenlet at 0x7ffa77017d70: <wal_e.worker.upload.WalUploader object at 0x7ffa76d6e750>(<wal_e.worker.pg.wal_transfer.WalSegment object at)> failed with timeout
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: wal_e.main   CRITICAL MSG: An unprocessed exception has avoided all error handling
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: DETAIL: Traceback (most recent call last):
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/cmd.py", line 585, in main
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: concurrency=args.pool_size)
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/operator/backup.py", line 282, in wal_archive
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: group.join()
Nov 05 01:26:06 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: File "/usr/local/lib/python2.7/dist-packages/wal_e/worker/pg/wal_transfer.py", line 148, in join
Nov 05 01:26:07 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: raise val
Nov 05 01:26:07 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: timeout: timed out
Nov 05 01:26:07 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: STRUCTURED: time=2014-11-05T01:26:06.987237-00 pid=494
Nov 05 01:26:07 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:26:06 UTC LOG:  archive command failed with exit code 2
Nov 05 01:26:07 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:26:06 UTC DETAIL:  The failed archive command was: envdir /etc/wal-e.d/env wal-e wal-push pg_xlog/000000010000000000000001
Nov 05 01:26:08 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: wal_e.main   INFO     MSG: starting WAL-E
Nov 05 01:26:08 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: DETAIL: The subcommand is "wal-push".
Nov 05 01:26:08 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: STRUCTURED: time=2014-11-05T01:26:08.094510-00 pid=655
Nov 05 01:26:08 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: wal_e.worker.upload INFO     MSG: begin archiving a file
Nov 05 01:26:08 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: DETAIL: Uploading "pg_xlog/000000010000000000000001" to "s3://db_wal/wal_005/000000010000000000000001.lzo".
Nov 05 01:26:08 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: STRUCTURED: time=2014-11-05T01:26:08.108945-00 pid=655 action=push-wal key=s3://db_wal/wal_005/000000010000000000000001.lzo prefix= seg=000000010000000000000001 state=begin
Nov 05 01:26:39 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:26:39 UTC FATAL:  password authentication failed for user "deis"
Nov 05 01:26:39 deis-d6c34852-3ca1-444c-9148-ceef87a902d1.novalocal sh[5989]: 2014-11-05 01:26:39 UTC DETAIL:  Connection matched pg_hba.conf line 102: "host         all                         all                         0.0.0.0/0                                 md5"