Created
December 17, 2022 21:47
-
-
Save Cactiw/c2397561966e2f343b0563a6c5b7f17f to your computer and use it in GitHub Desktop.
This is quick fix for sentry generated backup.json file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import json | |
| FILENAME = "backup.json" | |
| FILENAME_FIXED = "backup_fixed.json" | |
| def restore(): | |
| with open(FILENAME, "r") as f: | |
| lines = f.readlines() | |
| lines = lines[lines.index('[\n'):] | |
| with open(FILENAME_FIXED, "w") as f: | |
| f.writelines(lines) | |
| with open(FILENAME_FIXED, "r") as f: | |
| data = json.load(f) | |
| for d in data: | |
| value = d.get("fields", {}).get("value") | |
| if value is not None and not isinstance(value, str): | |
| d["fields"]["value"] = str(value) | |
| with open(FILENAME_FIXED, "w") as f: | |
| f.write(json.dumps(data, indent=4, ensure_ascii=False)) | |
| return data | |
| if __name__ == "__main__": | |
| restore() |
Still have some problems:
6:15:43 [INFO] sentry.plugins.github: apps-not-configured Traceback (most recent call last): File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 91, in inner return func(self, sql, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/base.py", line 86, in execute return self.cursor.execute(sql, clean_bad_params(params)) psycopg2.errors.UniqueViolation: duplicate key value violates unique constraint "sentry_option_key_key" DETAIL: Key (key)=(sentry:last_worker_ping) already exists. During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute return self.cursor.execute(sql, params) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 77, in inner raise_the_exception(self.db, e) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 75, in inner return func(self, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 18, in inner return func(self, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 95, in inner raise exc_info[0](msg).with_traceback(exc_info[2]) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 91, in inner return func(self, sql, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/base.py", line 86, in execute return self.cursor.execute(sql, clean_bad_params(params)) psycopg2.errors.UniqueViolation: UniqueViolation('duplicate key value violates unique constraint "sentry_option_key_key"\nDETAIL: Key (key)=(sentry:last_worker_ping) already exists.\n') SQL: UPDATE "sentry_option" SET "key" = %s, "value" = %s, "last_updated" = %s WHERE "sentry_option"."id" = %s The above exception was the direct cause of the following exception: Traceback (most recent call last): File "/usr/local/bin/sentry", line 8, in <module> sys.exit(main()) File "/usr/local/lib/python3.8/site-packages/sentry/runner/__init__.py", line 192, in main func(**kwargs) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1128, in __call__ return self.main(*args, **kwargs) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1053, in main rv = self.invoke(ctx) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1659, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 1395, in invoke return ctx.invoke(self.callback, **ctx.params) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/usr/local/lib/python3.8/site-packages/click/decorators.py", line 26, in new_func return f(get_current_context(), *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/runner/decorators.py", line 29, in inner return ctx.invoke(f, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/click/core.py", line 754, in invoke return __callback(*args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/runner/commands/backup.py", line 21, in import_ obj.save() File "/usr/local/lib/python3.8/site-packages/django/core/serializers/base.py", line 223, in save models.Model.save_base(self.object, using=using, raw=True, **kwargs) File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 780, in save_base updated = self._save_table( File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 853, in _save_table updated = self._do_update(base_qs, using, pk_val, values, update_fields, File "/usr/local/lib/python3.8/site-packages/django/db/models/base.py", line 903, in _do_update return filtered._update(values) > 0 File "/usr/local/lib/python3.8/site-packages/django/db/models/query.py", line 760, in _update return query.get_compiler(self.db).execute_sql(CURSOR) File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1471, in execute_sql cursor = super().execute_sql(result_type) File "/usr/local/lib/python3.8/site-packages/django/db/models/sql/compiler.py", line 1142, in execute_sql cursor.execute(sql, params) File "/usr/local/lib/python3.8/site-packages/sentry_sdk/integrations/django/__init__.py", line 563, in execute return real_execute(self, sql, params) File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 67, in execute return self._execute_with_wrappers(sql, params, many=False, executor=self._execute) File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 76, in _execute_with_wrappers return executor(sql, params, many, context) File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute return self.cursor.execute(sql, params) File "/usr/local/lib/python3.8/site-packages/django/db/utils.py", line 89, in __exit__ raise dj_exc_value.with_traceback(traceback) from exc_value File "/usr/local/lib/python3.8/site-packages/django/db/backends/utils.py", line 84, in _execute return self.cursor.execute(sql, params) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 77, in inner raise_the_exception(self.db, e) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 75, in inner return func(self, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 18, in inner return func(self, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 95, in inner raise exc_info[0](msg).with_traceback(exc_info[2]) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/decorators.py", line 91, in inner return func(self, sql, *args, **kwargs) File "/usr/local/lib/python3.8/site-packages/sentry/db/postgres/base.py", line 86, in execute return self.cursor.execute(sql, clean_bad_params(params)) django.db.utils.IntegrityError: UniqueViolation('duplicate key value violates unique constraint "sentry_option_key_key"\nDETAIL: Key (key)=(sentry:last_worker_ping) already exists.\n') SQL: UPDATE "sentry_option" SET "key" = %s, "value" = %s, "last_updated" = %s WHERE "sentry_option"."id" = %s
It seems like your target database is not empty, maybe Sentry has already recorded something there?
I also had to remove every sentry.useremail block from the backup file.
Error while importing to an empty database:
django.db.utils.IntegrityError: UniqueViolation('duplicate key value violates unique constraint "sentry_useremail_user_id_email_ade975f1_uniq"\nDETAIL: Key (user_id, email)=(3, test@example.com) already exists.\n')
SQL: UPDATE "sentry_useremail" SET "user_id" = %s, "email" = %s, "validation_hash" = %s, "date_hash_added" = %s, "is_verified" = %s WHERE "sentry_useremail"."id" = %s
Email addresses are probably already added by the sentry.user backups...
This script caused us issues with boolean values being converted to uppercase string "False". The import didn't like them.
I've adapted the script to account for this:
import json
FILENAME = "backup.json"
FILENAME_FIXED = "backup_fixed_2.json"
def restore():
with open(FILENAME, "r") as f:
lines = f.readlines()
lines = lines[lines.index('[\n'):]
with open(FILENAME_FIXED, "w") as f:
f.writelines(lines)
with open(FILENAME_FIXED, "r") as f:
data = json.load(f)
for d in data:
value = d.get("fields", {}).get("value")
if isinstance(value, bool):
d["fields"]["value"] = str(value).lower()
elif value is not None and not isinstance(value, str):
d["fields"]["value"] = str(value)
with open(FILENAME_FIXED, "w") as f:
f.write(json.dumps(data, indent=4, ensure_ascii=False))
return data
if __name__ == "__main__":
restore()
This script is broken and imports booleans as (encoded) strings into the database.
A True should be encoded as "gAKILg==" but gets inserted as "gAJYBAAAAFRydWVxAC4=" into the database.
After hours of debugging i finally managed to fix my installation with multiple manual db queries.
Sadly i dont have every query in my history, but some are here:
update sentry_option set value = 'gAKILg==' where value = 'gAJYBAAAAFRydWVxAC4=';
update sentry_organizationoptions set value = 'gAKILg==' where value = 'gAJYBAAAAFRydWVxAC4=';
update sentry_projectoptions set value = 'gAKILg==' where value = 'gAJYBAAAAFRydWVxAC4=';
@Lucaber , my adapted script above should be of help if you have your original exported backup.
If i remember correctly, numbers were also saved incorrectly as base64 encoded strings with quotes. 8 -> "8" | base64.
This resulted in broken "sentry:option-epoch" options.
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Still have some problems: