CKAN errors help

gistfile1.txt

Lots of stack traces and error messages for CKAN, followed by a comment about how I fixed it (that time).
NB don't email me privately for help - ask on StackOverflow #ckan instead.
David

File "/vagrant/src/ckan/ckan/templates/home/snippets/search.html", line 1, in top-level template code
    {% set tags = h.get_facet_items_dict('tags', limit=3) %}
  File "/vagrant/src/ckan/ckan/lib/helpers.py", line 949, in get_facet_items_dict
    if not c.search_facets or \
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/werkzeug/local.py", line 347, in __getattr__
    return getattr(self._get_current_object(), name)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/werkzeug/local.py", line 347, in __getattr__
    return getattr(self._get_current_object(), name)
  File "/vagrant/src/ckan/ckan/config/middleware/flask_app.py", line 316, in __getattr__
    return getattr(app_globals.app_globals, name)
AttributeError: '_Globals' object has no attribute 'search_facets'
# fixed by restarting jetty - it was not working.

  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/nose/plugins/manager.py", line 167, in simple
    result = meth(*arg, **kw)
  File "/usr/lib/ckan/default/src/ckan/ckan/ckan_nose_plugin.py", line 20, in startContext
    import ckan.model as model
AttributeError: 'module' object has no attribute 'model'
# fixed this error  (on gds laptop) by doing:
#	cd /vagrant/src/ckan
# python setup.py develop

ProgrammingError: (psycopg2.ProgrammingError) relation "_table_metadata" does not exist
# fixed by running set-permissions command:
# paster --plugin=ckan datastore set-permissions -c test-core.ini | sudo -u postgres psql --set ON_ERROR_STOP=1

  File "/vagrant/src/ckan/ckanext/datastore/plugin.py", line 13, in <module>
    import ckanext.datastore.logic.action as action
  File "/vagrant/src/ckan/ckanext/datastore/logic/action.py", line 19, in <module>
    from ckanext.datastore.backend.postgres import identifier
  File "/vagrant/src/ckan/ckanext/datastore/backend/postgres.py", line 28, in <module>
    from psycopg2.extras import register_default_json, register_composite
ImportError: cannot import name register_default_json
# fixed by: pip install requirements.txt
# probably for: psycopg2-2.7.3.2

  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/httpretty/core.py", line 523, in validate
    if igot > self.body_length:
UnboundLocalError: local variable 'igot' referenced before assignment
# fixed by upgrading httpretty-0.8.3 -> 0.8.14

  File "/vagrant/src/ckan/ckan/migration/versions/008_update_vdm_ids.py", line 44, in drop_constraints_and_alter_types
    id_col = constraint.table.columns[constraint.columns[0]]
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/sqlalchemy/util/_collections.py", line 194, in __getitem__
    return self._data[key]
KeyError: 0
# wrong version of sqlalchemy - pip install -r requirements.txt

  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/migrate/versioning/version.py", line 207, in script
    "There is no script for %d version" % self.version
AssertionError: There is no script for 86 version
# stray pyc files
# find . -name "*.pyc"|xargs rm

  File "/vagrant/src/ckan/ckan/views/user.py", line 151, in read
  File "/vagrant/src/ckan/ckan/lib/base.py", line 193, in render
    return flask_render_template(template_name, **extra_vars)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/flask/templating.py", line 134, in render_template
    context, ctx.app)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/flask/templating.py", line 116, in _render
    rv = template.render(context)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/jinja2/environment.py", line 989, in render
    return self.environment.handle_exception(exc_info, True)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/jinja2/environment.py", line 754, in handle_exception
    reraise(exc_type, exc_value, tb)
  File "/vagrant/src/ckan/ckan/templates/user/read.html", line 1, in top-level template code
    {% extends "user/read_base.html" %}
  File "/vagrant/src/ckan/ckan/templates/user/read_base.html", line 3, in top-level template code
    {% set user = c.user_dict %}
  File "/vagrant/src/ckan/ckan/templates/page.html", line 1, in top-level template code
    {% extends "base.html" %}
  File "/vagrant/src/ckan/ckan/templates/base.html", line 41, in top-level template code
    {%- block title -%}
  File "/vagrant/src/ckan/ckan/templates/base.html", line 42, in block "title"
    {%- block subtitle %}{% endblock -%}
  File "/vagrant/src/ckan/ckan/templates/user/read_base.html", line 5, in block "subtitle"
    {% block subtitle %}{{ user.display_name }} - {{ _('Users') }}{% endblock %}
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/jinja2/environment.py", line 408, in getattr
    return getattr(obj, attribute)
UndefinedError: 'werkzeug.local.LocalProxy object' has no attribute 'user_dict'
# stray pyc files
# find . -name "*.pyc"|xargs rm

When installing requirements.txt for 2.5.7:
 ImportError: No module named testrepository
# Apparently it's to do with setuptools version.
# $ easy_install --version
# setuptools 36.1.0 from /usr/lib/ckan/default/lib/python2.7/site-packages (Python 2.7)
# https://bugs.launchpad.net/tempest/+bug/1530026
# suggests its a bug exposed by setuptools changes
# pip install testrepository
# pip install -r requirements.txt

LoaderError: Could not create the database table: (psycopg2.ProgrammingError) function populate_full_text_trigger() does not exist
# paster --plugin=ckan datastore set-permissions | sudo -u postgres psql --set ON_ERROR_STOP=1
# paster --plugin=ckan datastore set-permissions -c ../ckanext-xloader/test.ini| sudo -u postgres psql --set ON_ERROR_STOP=1

nosetests --ckan --with-pylons=test-core.ini ckanext/datastore/tests/test_dump.py:TestDatastoreDump.test_dump_basic -s
...
ProgrammingError: (psycopg2.ProgrammingError) function populate_full_text_trigger() does not exist
 [SQL: 'CREATE TRIGGER zfulltext\n        BEFORE INSERT OR UPDATE ON "b761c191-3534-4b30-84d6-d515347f0eed"\n        FOR EACH ROW EXECUTE PROCEDURE populate_full_text_trigger()']
# needed to run:
# sudo -u postgres psql datastore_default -f ../ckanext-xloader/full_text_function.sql
# sudo -u postgres psql datastore_test -f ../ckanext-xloader/full_text_function.sql


paster --plugin=ckanext-rq jobs -c $CKAN_INI list
  File "/usr/lib/ckan/default/src/ckanext-rq/ckanext/rq/cli.py", line 86, in command
    self.list()
  File "/usr/lib/ckan/default/src/ckanext-rq/ckanext/rq/cli.py", line 106, in list
    jobs = p.toolkit.get_action(u'job_list')({}, data_dict)
  File "/usr/lib/ckan/default/src/ckan/ckan/logic/__init__.py", line 347, in get_action
    raise KeyError("Action '%s' not found" % action)
KeyError: u"Action 'job_list' not found"
# fixed by adding rq to the ckan.plugins

nosetests --with-pylons test.ini ckanext/shift/tests/test_jobs.py -s
ERROR: ckanext.shift.tests.test_jobs.TestShiftDataIntoDatastore.test_simple_csv
  File "/vagrant/src/ckanext-shift/ckanext/shift/tests/test_jobs.py", line 27, in test
    requests.get(u'http://127.0.0.1:8983/solr/ckan/select/?Error during load: Could not create the database table: SOLR returned an error running query: {'fq': ['site_id:"test.ckan.net"', u'+site_id:"test.ckan.net"', '+state:active'], 'facet.mincount': 1, 'rows': 2, 'facet': 'true', 'q': 'id:"31ea345d-02c1-4e7d-b64e-f4d9509b99cc"', 'facet.limit': '50', 'wt': 'json', 'fl': 'data_dict'} Error: SolrError(u'Failed to connect to server at \'http://127.0.0.1:8983/solr/ckan/select/?fq=site_id%3A%22test.ckan.net%22&fq=%2Bsite_id%3A%22test.ckan.net%22&fq=%2Bstate%3Aactive&facet.mincount=1&rows=2&facet=true&q=id%3A%2231ea345d-02c1-4e7d-b64e-f4d9509b99cc%22&facet.limit=50&wt=json&fl=data_dict\', are you sure that URL is correct? Checking it in a browser might help: (\'Connection aborted.\', BadStatusLine("\'\'",))',)
# Due to httpretty causing unregistered urls to not work, such as this call to SOLR. Maybe to do incompat with requests version.
# We have httpretty==0.8.14 (latest). Trying older one suggested:
# pip install httpretty==0.6.4
# appears to fix it

sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) could not connect to server: Connection refused
# CKAN logs show it couldn't connect to the db - it wasn't ready that time 

sqlalchemy.exc.ProgrammingError: (psycopg2.ProgrammingError) column "id" referenced in foreign key constraint does not exist" during "upgrade_db"
# issue: https://github.com/ckan/ckan/issues/3929
# added one-liner - https://github.com/ckan/ckan/pull/4016

ERROR: no CKAN_SQLALCHEMY_URL specified in docker-compose.yml
# no bind mount specified, so not surprised it fails

ImportError: cannot import name http_cookies
docker run -it --entrypoint=/bin/bash -v /Users/davidread/ckan/volumes/ckan_home:/usr/lib/ckan docker_ckan
. /usr/lib/ckan/venv/bin/activate
# requirements.txt:
# Either: change the build-copy, rebuild and then re-copy the venv to your local bind-mounted dir OR change the copy in the local/bind-mounted dir, log-in and ckan-pip install it
# I did the latter
# Upgrading beaker to 1.9.0 fixed the http_cookies problem

  File "/usr/lib/ckan/venv/src/ckan/ckan/lib/redis.py", line 14, in <module>
    from redis import ConnectionPool, Redis
  File "/usr/lib/ckan/venv/src/ckan/ckan/lib/redis.py", line 14, in <module>
    from redis import ConnectionPool, Redis
ImportError: cannot import name ConnectionPool
# i.e. circular import. NOT to do with macos being case insensitive (linux is not)
# is because ckan/lib is in the python path, so ckan's redis.py is getting picked ahead of the redis library module

  File "/vagrant/src/ckanext-fluent/ckanext/fluent/helpers.py", line 3, in <module>
    from ckanext.scheming.helpers import scheming_language_text
ImportError: No module named scheming.helpers
# Needs ckanext-scheming

 Module /vagrant/src/ckanext-bostonschema/ckanext/bostonschema/templates/scheming/package/read.html:14 in block "package_notes"
>>  {% set view = h.get_canonical_resource_view(pkg.id) %}
Module jinja2.environment:412 in getattr         view
>>  return obj[attribute]
Module ckan.lib.helpers:79 in __getitem__         view
>>  key=key
HelperError: Helper 'get_canonical_resource_view' has not been defined.
# install ckanext-featuredviews

File '/vagrant/src/ckan/ckan/controllers/package.py', line 421 in read
  'is_activity_archive': bool(activity_id),
File '/vagrant/src/ckan/ckan/lib/base.py', line 195 in render
  return cached_template(template_name, render_template)
File '/usr/lib/ckan/default/local/lib/python2.7/site-packages/pylons/templating.py', line 249 in cached_template
  return render_func()
File '/vagrant/src/ckan/ckan/lib/base.py', line 137 in render_template
  return render_jinja2(template_name, globs)
File '/vagrant/src/ckan/ckan/lib/base.py', line 94 in render_jinja2
  return template.render(**extra_vars)
File '/usr/lib/ckan/default/local/lib/python2.7/site-packages/jinja2/environment.py', line 989 in render
  return self.environment.handle_exception(exc_info, True)
File '/usr/lib/ckan/default/local/lib/python2.7/site-packages/jinja2/environment.py', line 754 in handle_exception
  reraise(exc_type, exc_value, tb)
File '/vagrant/src/ckanext-bostonschema/ckanext/bostonschema/templates/scheming/package/read.html', line 1 in top-level template code
  {% ckan_extends %}
File '/vagrant/src/ckanext-scheming/ckanext/scheming/templates/scheming/package/read.html', line 3 in top-level template code
  {%- set schema = h.scheming_get_dataset_schema(dataset_type) -%}
File '/usr/lib/ckan/default/local/lib/python2.7/site-packages/jinja2/environment.py', line 412 in getattr
  return obj[attribute]
File '/vagrant/src/ckan/ckan/lib/helpers.py', line 79 in __getitem__
  key=key
HelperError: Helper 'scheming_get_dataset_schema' has not been defined.
# Fixed by enabling plugin: scheming_datasets

$ paster --plugin=abc
Traceback (most recent call last):
  File "/usr/lib/ckan/default/bin/paster", line 11, in <module>
    sys.exit(run())
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/paste/script/command.py", line 91, in run
    commands = get_commands()
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/paste/script/command.py", line 133, in get_commands
    plugins = pluginlib.resolve_plugins(plugins)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/paste/script/pluginlib.py", line 83, in resolve_plugins
    pkg_resources.require(plugin)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/pkg_resources/__init__.py", line 971, in require
    needed = self.resolve(parse_requirements(requirements))
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/pkg_resources/__init__.py", line 857, in resolve
    raise DistributionNotFound(req, requirers)
pkg_resources.DistributionNotFound
# I got the name of the plugin wrong and it can't find it. should be 'ckan' or 'ckanext-harvest' etc

$ paster --plugin=ckan datapusher submit periodic-groundwater-level-measurements
Data in any datastore resource that isn't in their source files (e.g. data added using the datastore API) will be permanently lost. Are you sure you want to proceed? [y/n] y
Submitting 1 datastore resources
Traceback (most recent call last):
  File "/usr/lib/ckan/default/bin/paster", line 11, in <module>
    sys.exit(run())
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/paste/script/command.py", line 102, in run
    invoke(command, command_name, options, args[1:])
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/paste/script/command.py", line 141, in invoke
    exit_code = runner.run(args)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/paste/script/command.py", line 236, in run
    result = self.command()
  File "/vagrant/src/ckan/ckanext/datapusher/cli.py", line 59, in command
    self._submit_package(self.args[1])
  File "/vagrant/src/ckan/ckanext/datapusher/cli.py", line 100, in _submit_package
    self._submit(resource_ids)
  File "/vagrant/src/ckan/ckanext/datapusher/cli.py", line 108, in _submit
    datapusher_submit = p.toolkit.get_action('datapusher_submit')
  File "/vagrant/src/ckan/ckan/logic/__init__.py", line 384, in get_action
    raise KeyError("Action '%s' not found" % action)
KeyError: "Action 'datapusher_submit' not found"
# Fixed by enabling the ckanext-datapusher extension, by adding it to my development.ini's ckan.plugins:
# ckan.plugins = stats text_view image_view recline_view datastore datapusher

End of script output before headers: apache.wsgi
# (seen in the apache errors log)
# Occurs straight as soon as CKAN talks to postgres, in this line in model/__init__.py:
#     version_table = Table('migrate_version', meta.metadata, autoload=True)
# I don't know what the problem was - the apache config looked fine. But I fixed it by completely reinstalling apache2:
# sudo apt-get remove apache2
# sudo mv /etc/apache2 /etc/apache2-old
# sudo apt-get purge apache2
# sudo apt-get install apache2

Error viewing a dataset:
  File "/vagrant/src/ckan/ckan/lib/helpers.py", line 1758, in follow_button
    following = logic.get_action(action)(context, {'id': obj_id})
  File "/vagrant/src/ckan/ckan/logic/__init__.py", line 464, in wrapped
    result = _action(context, data_dict, **kw)
  File "/vagrant/src/ckan/ckan/logic/action/get.py", line 2872, in am_following_dataset
    context['model'].UserFollowingDataset)
  File "/vagrant/src/ckan/ckan/logic/action/get.py", line 2838, in _am_following
    raise logic.NotAuthorized
    # This particular NotAuthorized is caused by being logged in as a user, then that user is wiped from the database (e.g. restoring the db from a backup) and then you view a dataset.
    # Fixed by logging out (even though it doesn't say you are logged in): browse to /user/logout
    
At: /dataset/
  File "/vagrant/src/ckan/ckan/templates/snippets/package_item.html", line 36, in block "heading_title"
    {{ h.link_to(h.truncate(title, truncate_title), h.url_for('%s.read' % package.type, id=package.name)) }}
  File "/vagrant/src/ckan/ckan/lib/helpers.py", line 324, in url_for
    my_url = _url_for_pylons(*args, **kw)
  File "/vagrant/src/ckan/ckan/lib/helpers.py", line 413, in _url_for_pylons
    return _routes_default_url_for(*args, **kw)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/routes/util.py", line 265, in url_for
    "unicode instead: %s" % url)
GenerationException: url_for can only return a string, got unicode instead: harvest.read?id=csw-test
# Due to trying to display a 'harvest' type dataset. Should not be included in search.
# CKAN 1.9+'s dataset view changes default and returns all datasets types.
# Harvest extension uses before_search to filter out harvest type datasets.
# Fixed by enabling harvest extension in production.ini's ckan.plugins.

nosetests --with-pylons test.ini --ckan ../ckanext-xloader/ckanext/xloader/tests
  File "/vagrant/src/ckanext-xloader/ckanext/xloader/tests/test_action.py", line 17, in setup_class
    p.load('datastore')
  File "/vagrant/src/ckan/ckan/plugins/core.py", line 168, in load
    plugins_update()
  File "/vagrant/src/ckan/ckan/plugins/core.py", line 122, in plugins_update
    environment.update_config()
  File "/vagrant/src/ckan/ckan/config/environment.py", line 298, in update_config
    plugin.configure(config)
  File "/vagrant/src/ckan/ckanext/datastore/plugin.py", line 80, in configure
    self.backend.configure(config)
  File "/vagrant/src/ckan/ckanext/datastore/backend/postgres.py", line 1695, in configure
    if self._is_read_only_database():
  File "/vagrant/src/ckan/ckanext/datastore/backend/postgres.py", line 1613, in _is_read_only_database
    connection = _get_engine_from_url(url).connect()
  File "/vagrant/src/ckan/ckanext/datastore/backend/postgres.py", line 128, in _get_engine_from_url
    loads=lambda x: x)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/psycopg2/_json.py", line 165, in register_default_json
    loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/psycopg2/_json.py", line 147, in register_json
    register_type(JSON, not globally and conn_or_curs or None)
TypeError: argument 2 must be a connection, cursor or None
# due to using sqlite database, instead of the postgres these tests were expecting. switch to test-core.ini:
# nosetests --with-pylons test-core.ini --ckan ../ckanext-xloader/ckanext/xloader/tests

$ nosetests --ckan --with-pylons=test-core.ini ckan/tests/test_coding_standards.py:test_building_the_docs
======================================================================
ERROR: test suite for <module 'ckan.tests.test_coding_standards' from '/vagrant/src/ckan/ckan/tests/test_coding_standards.py'>
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/nose/suite.py", line 209, in run
    self.setUp()
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/nose/suite.py", line 292, in setUp
    self.setupContext(ancestor)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/nose/suite.py", line 301, in setupContext
    self.config.plugins.startContext(context)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/nose/plugins/manager.py", line 99, in __call__
    return self.call(*arg, **kw)
  File "/usr/lib/ckan/default/local/lib/python2.7/site-packages/nose/plugins/manager.py", line 167, in simple
    result = meth(*arg, **kw)
  File "/usr/lib/ckan/default/src/ckan/ckan/ckan_nose_plugin.py", line 20, in startContext
    import ckan.model as model
AttributeError: 'module' object has no attribute 'model'
# fixed by reinstalling ckan source into the virtual environment:
# pip install -e /vagrant/src/ckan