Skip to content

Instantly share code, notes, and snippets.

@aronasorman

aronasorman/gist:e793f94d40c951ca069a

Last active August 29, 2015 14:24
Show Gist options
  • Save aronasorman/e793f94d40c951ca069a to your computer and use it in GitHub Desktop.
Save aronasorman/e793f94d40c951ca069a to your computer and use it in GitHub Desktop.
Download ZIP
Raw
gistfile1.txt
Line # Mem usage Increment Line Contents
================================================
391 28.4 MiB 0.0 MiB @profile
392 def start(debug=False, daemonize=True, args=[], skip_job_scheduler=False, port=None):
393 """
394 Start the kalite server as a daemon
395
396 :param args: List of options to parse to the django management command
397 :param port: Non-default port to bind to. You cannot run kalite on
398 multiple ports at the same time.
399 :param daemonize: Default True, will run in foreground if False
400 :param skip_job_scheduler: Skips running the job scheduler in a separate thread
401 """
402 # TODO: Do we want to fail if running as root?
403
404 28.4 MiB 0.0 MiB port = int(port or DEFAULT_LISTEN_PORT)
405
406 28.4 MiB 0.0 MiB if not daemonize:
407 28.4 MiB 0.0 MiB sys.stderr.write("Running 'kalite start' in foreground...\n")
408 else:
409 sys.stderr.write("Running 'kalite start' as daemon (system service)\n")
410
411 28.4 MiB 0.0 MiB sys.stderr.write("\nStand by while the server loads its data...\n\n")
412
413 28.4 MiB 0.0 MiB if os.path.exists(STARTUP_LOCK):
414 try:
415 pid, __ = read_pid_file(STARTUP_LOCK)
416 # Does the PID in there still exist?
417 if pid_exists(pid):
418 sys.stderr.write(
419 "Refusing to start: Start up lock exists: {0:s}\n".format(STARTUP_LOCK))
420 sys.exit(1)
421 # Couldn't parse to int
422 except TypeError:
423 pass
424
425 os.unlink(STARTUP_LOCK)
426
427 28.4 MiB 0.0 MiB try:
428 28.4 MiB 0.0 MiB if get_pid():
429 sys.stderr.write("Refusing to start: Already running\n")
430 sys.exit(1)
431 28.4 MiB 0.0 MiB except NotRunning:
432 28.4 MiB 0.0 MiB pass
433
434 # Write current PID and optional port to a startup lock file
435 28.4 MiB 0.0 MiB with open(STARTUP_LOCK, "w") as f:
436 28.4 MiB 0.0 MiB f.write("%s\n%d" % (str(os.getpid()), port))
437
438 542.6 MiB 514.2 MiB manage('initialize_kalite')
439
440 # Start the job scheduler (not Celery yet...)
441 # This command is run before starting the server, in case the server
442 # should be configured to not run in daemon mode or in case the
443 # server fails to go to daemon mode.
444 542.6 MiB 0.0 MiB if not skip_job_scheduler:
445 542.6 MiB 0.0 MiB manage(
446 542.6 MiB 0.0 MiB 'cronserver_blocking',
447 542.6 MiB 0.0 MiB args=[],
448 542.7 MiB 0.1 MiB as_thread=True
449 )
450
451 # Remove the startup lock at this point
452 542.9 MiB 0.2 MiB if STARTUP_LOCK:
453 542.9 MiB 0.0 MiB os.unlink(STARTUP_LOCK)
454
455 # Print output to user about where to find the server
456 542.9 MiB 0.0 MiB addresses = get_ip_addresses(include_loopback=False)
457 542.9 MiB 0.0 MiB sys.stdout.write("To access KA Lite from another connected computer, try the following address(es):\n")
458 542.9 MiB 0.0 MiB for addr in addresses:
459 542.9 MiB 0.0 MiB sys.stdout.write("\thttp://%s:%s/\n" % (addr, port))
460 542.9 MiB 0.0 MiB sys.stdout.write("To access KA Lite from this machine, try the following address:\n")
461 542.9 MiB 0.0 MiB sys.stdout.write("\thttp://127.0.0.1:%s/\n" % port)
462
463 # Daemonize at this point, no more user output is needed
464 542.9 MiB 0.0 MiB if daemonize:
465
466 from django.utils.daemonize import become_daemon
467 kwargs = {}
468 # Truncate the file
469 open(SERVER_LOG, "w").truncate()
470 print("Going to daemon mode, logging to {0}".format(SERVER_LOG))
471 kwargs['out_log'] = SERVER_LOG
472 kwargs['err_log'] = SERVER_LOG
473 become_daemon(**kwargs)
474 # Write the new PID
475 with open(PID_FILE, 'w') as f:
476 f.write("%d\n%d" % (os.getpid(), port))
477
478 # Start cherrypy service
479 542.9 MiB 0.0 MiB cherrypy.config.update({
480 542.9 MiB 0.0 MiB 'server.socket_host': LISTEN_ADDRESS,
481 542.9 MiB 0.0 MiB 'server.socket_port': port,
482 542.9 MiB 0.0 MiB 'server.thread_pool': 18,
483 542.9 MiB 0.0 MiB 'checker.on': False,
484 })
485
486 542.9 MiB 0.0 MiB DjangoAppPlugin(cherrypy.engine).subscribe()
487 542.9 MiB 0.0 MiB if not debug:
488 # cherrypyserver automatically reloads if any modules change
489 # Switch-off that functionality here to save cpu cycles
490 # http://docs.cherrypy.org/stable/appendix/faq.html
491 542.9 MiB 0.0 MiB cherrypy.engine.autoreload.unsubscribe()
492
493 547.3 MiB 4.5 MiB cherrypy.quickstart()
494
495 547.3 MiB 0.0 MiB print("FINISHED serving HTTP")
Raw
gistfile2.txt
Line # Mem usage Increment Line Contents
================================================
39 51.3 MiB 0.0 MiB @profile
40 def reinitialize_server(self):
41 """Reset the server state."""
42 51.3 MiB 0.0 MiB logging.info("Invalidating the web cache.")
43 51.3 MiB 0.0 MiB from fle_utils.internet.webcache import invalidate_web_cache
44 51.3 MiB 0.0 MiB invalidate_web_cache()
45
46 # Next, call videoscan.
47 51.3 MiB 0.0 MiB logging.info("Running videoscan.")
48 51.9 MiB 0.6 MiB call_command("videoscan")
49
50 # Finally, pre-load global data
51 542.6 MiB 490.7 MiB initialize_content_caches()
Line # Mem usage Increment Line Contents
================================================
53 50.4 MiB 0.0 MiB @profile
54 def handle(self, *args, **options):
55
56 51.3 MiB 0.9 MiB self.setup_server_if_needed()
57
58 # we do this on every server request,
59 # as we don't know what happens when we're not looking.
60 542.6 MiB 491.2 MiB self.reinitialize_server()
61
62 # Copy static media, one reason for not symlinking: It is not cross-platform and can cause permission issues
63 # with many webservers
64 542.6 MiB 0.0 MiB logging.info("Copying static media...")
65 542.6 MiB 0.0 MiB call_command("collectstatic", interactive=False, verbosity=0)
66 542.8 MiB 0.2 MiB call_command("collectstatic_js_reverse", interactive=False)
Raw
gistfile3.txt
Line # Mem usage Increment Line Contents
================================================
74 51.8 MiB 0.0 MiB @profile
75 def initialize_content_caches(force=False):
76 """
77 Catch all function to regenerate any content caches in memory that need annotation
78 with file availability
79 """
80 542.2 MiB 490.4 MiB for lang in i18n.get_installed_language_packs(force=True).keys():
81 411.4 MiB -130.8 MiB logging.info("Preloading exercise data for language {lang}.".format(lang=lang))
82 448.2 MiB 36.7 MiB topic_tools.get_exercise_cache(force=force, language=lang)
83 448.2 MiB 0.0 MiB logging.info("Preloading content data for language {lang}.".format(lang=lang))
84 495.8 MiB 47.6 MiB topic_tools.get_content_cache(force=force, annotate=True, language=lang)
85 495.8 MiB 0.0 MiB logging.info("Preloading topic tree data for language {lang}.".format(lang=lang))
86 542.2 MiB 46.4 MiB topic_tools.get_topic_tree(force=force, annotate=True, language=lang)
Raw
gistfile4.txt
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/i18n/__init__.py
Line # Mem usage Increment Line Contents
================================================
306 51.8 MiB 0.0 MiB @profile
307 def get_installed_language_packs(force=False):
308 global INSTALLED_LANGUAGES_CACHE
309 51.8 MiB 0.0 MiB if not INSTALLED_LANGUAGES_CACHE or force:
310 51.8 MiB 0.0 MiB INSTALLED_LANGUAGES_CACHE = _get_installed_language_packs()
311 51.8 MiB 0.0 MiB return INSTALLED_LANGUAGES_CACHE
INFO:kalite:Preloading exercise data for language en.
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
166 51.9 MiB 0.0 MiB @profile
167 def get_exercise_cache(force=False, language=None):
168
169 51.9 MiB 0.0 MiB if not language:
170 language = settings.LANGUAGE_CODE
171
172 global EXERCISES, EXERCISES_FILEPATH
173 51.9 MiB 0.0 MiB if EXERCISES is None:
174 51.9 MiB 0.0 MiB EXERCISES = {}
175 51.9 MiB 0.0 MiB if EXERCISES.get(language) is None:
176 51.9 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP and not force:
177 exercises = softload_json(EXERCISES_FILEPATH + "_" + language + ".cache", logger=logging.debug, raises=False)
178 if exercises:
179 EXERCISES[language] = exercises
180 return EXERCISES[language]
181 65.5 MiB 13.6 MiB EXERCISES[language] = softload_json(EXERCISES_FILEPATH, logger=logging.debug, raises=False)
182 65.5 MiB 0.0 MiB if language == "en": # English-language exercises live in application space, translations in user space
183 65.5 MiB 0.0 MiB exercise_root = os.path.join(settings.KHAN_EXERCISES_DIRPATH, "exercises")
184 else:
185 exercise_root = os.path.join(settings.USER_DATA_ROOT, "exercises")
186 65.5 MiB 0.0 MiB if os.path.exists(exercise_root):
187 65.5 MiB 0.0 MiB exercise_path = os.path.join(exercise_root, language) if language != "en" else exercise_root
188 65.5 MiB 0.0 MiB try:
189 65.5 MiB 0.0 MiB exercise_templates = os.listdir(exercise_path)
190 except OSError:
191 exercise_templates = []
192 else:
193 exercise_templates = []
194
195 98.6 MiB 33.1 MiB for exercise in EXERCISES[language].values():
196 98.6 MiB -0.0 MiB exercise_file = exercise["name"] + ".html"
197 98.6 MiB 0.0 MiB exercise_template = exercise_file
198 98.6 MiB 0.0 MiB exercise_lang = "en"
199
200 98.6 MiB 0.0 MiB if exercise.get("uses_assessment_items", False):
201 98.6 MiB 0.0 MiB available = False
202 98.6 MiB 0.0 MiB items = []
203 98.6 MiB 0.0 MiB for item in exercise.get("all_assessment_items","[]"):
204 98.6 MiB 0.0 MiB item = json.loads(item)
205 98.6 MiB 0.0 MiB if get_assessment_item_data(request=None, assessment_item_id=item.get("id")):
206 items.append(item)
207 available = True
208 98.6 MiB 0.0 MiB exercise["all_assessment_items"] = items
209 else:
210 98.5 MiB -0.1 MiB available = exercise_template in exercise_templates
211
212 # Get the language codes for exercise templates that exist
213 # Try to minimize the number of os.path.exists calls (since they're a bottleneck) by using the same
214 # precedence rules in i18n.select_best_available_languages
215 98.5 MiB 0.0 MiB available_langs = set(["en"] + [language]*available)
216 # Return the best available exercise template
217 98.5 MiB 0.0 MiB exercise_lang = i18n.select_best_available_language(language, available_codes=available_langs)
218
219 98.6 MiB 0.1 MiB if exercise_lang == "en":
220 98.6 MiB 0.0 MiB exercise_template = exercise_file
221 else:
222 exercise_template = os.path.join(exercise_lang, exercise_file)
223
224 98.6 MiB 0.0 MiB with i18n.translate_block(exercise_lang):
225 98.6 MiB 0.0 MiB exercise["available"] = available
226 98.6 MiB 0.0 MiB exercise["lang"] = exercise_lang
227 98.6 MiB 0.0 MiB exercise["template"] = exercise_template
228 98.6 MiB 0.0 MiB exercise["title"] = _(exercise.get("title", ""))
229 98.6 MiB 0.0 MiB exercise["description"] = _(exercise.get("description", "")) if exercise.get("description") else ""
230
231 98.6 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
232 try:
233 with open(EXERCISES_FILEPATH + "_" + language + ".cache", "w") as f:
234 json.dump(EXERCISES[language], f)
235 except IOError as e:
236 logging.warn("Annotated exercise cache file failed in saving with error {e}".format(e=e))
237
238 98.6 MiB 0.0 MiB return EXERCISES[language]
INFO:kalite:Preloading content data for language en.
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
263 98.6 MiB 0.0 MiB @profile
264 def get_content_cache(force=False, annotate=False, language=None):
265
266 98.6 MiB 0.0 MiB if not language:
267 language = settings.LANGUAGE_CODE
268
269 global CONTENT, CONTENT_FILEPATH
270
271 98.6 MiB 0.0 MiB if CONTENT is None:
272 98.6 MiB 0.0 MiB CONTENT = {}
273 98.6 MiB 0.0 MiB if CONTENT.get(language) is None:
274 144.6 MiB 46.1 MiB CONTENT[language] = softload_json(CONTENT_FILEPATH, logger=logging.debug, raises=False)
275 144.6 MiB 0.0 MiB annotate = True
276
277 144.6 MiB 0.0 MiB if annotate:
278 144.6 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP and not force:
279 content = softload_json(CONTENT_FILEPATH + "_" + language + ".cache", logger=logging.debug, raises=False)
280 if content:
281 CONTENT[language] = content
282 return CONTENT[language]
283
284 # Loop through all content items and put thumbnail urls, content urls,
285 # and subtitle urls on the content dictionary, and list all languages
286 # that the content is available in.
287 144.6 MiB 0.0 MiB try:
288 144.6 MiB 0.0 MiB contents_folder = os.listdir(settings.CONTENT_ROOT)
289 except OSError:
290 contents_folder = []
291
292 144.6 MiB 0.0 MiB subtitle_langs = {}
293
294 144.6 MiB 0.0 MiB if os.path.exists(i18n.get_srt_path()):
295 144.6 MiB 0.0 MiB for (dirpath, dirnames, filenames) in os.walk(i18n.get_srt_path()):
296 # Only both looking at files that are inside a 'subtitles' directory
297 144.6 MiB 0.0 MiB if os.path.basename(dirpath) == "subtitles":
298 144.6 MiB 0.0 MiB lc = os.path.basename(os.path.dirname(dirpath))
299 144.6 MiB 0.0 MiB for filename in filenames:
300 144.6 MiB 0.0 MiB if filename in subtitle_langs:
301 144.6 MiB 0.0 MiB subtitle_langs[filename].append(lc)
302 else:
303 144.6 MiB 0.0 MiB subtitle_langs[filename] = [lc]
304
305 161.6 MiB 17.0 MiB for content in CONTENT[language].values():
306 161.6 MiB -0.0 MiB default_thumbnail = create_thumbnail_url(content.get("id"))
307 161.6 MiB 0.0 MiB dubmap = i18n.get_id2oklang_map(content.get("id"))
308 161.6 MiB 0.0 MiB if dubmap:
309 161.6 MiB 0.0 MiB content_lang = i18n.select_best_available_language(language, available_codes=dubmap.keys()) or ""
310 161.6 MiB 0.0 MiB if content_lang:
311 161.6 MiB 0.0 MiB dubbed_id = dubmap.get(content_lang)
312 161.6 MiB 0.0 MiB format = content.get("format", "")
313 161.6 MiB 0.0 MiB if (dubbed_id + "." + format) in contents_folder:
314 159.9 MiB -1.7 MiB content["available"] = True
315 159.9 MiB 0.0 MiB thumbnail = create_thumbnail_url(dubbed_id) or default_thumbnail
316 159.9 MiB 0.0 MiB content["content_urls"] = {
317 159.9 MiB 0.0 MiB "stream": settings.CONTENT_URL + dubmap.get(content_lang) + "." + format,
318 159.9 MiB 0.0 MiB "stream_type": "{kind}/{format}".format(kind=content.get("kind", "").lower(), format=format),
319 159.9 MiB 0.0 MiB "thumbnail": thumbnail,
320 }
321 161.6 MiB 1.7 MiB elif settings.BACKUP_VIDEO_SOURCE:
322 content["available"] = True
323 content["content_urls"] = {
324 "stream": settings.BACKUP_VIDEO_SOURCE.format(youtube_id=dubbed_id, video_format=format),
325 "stream_type": "{kind}/{format}".format(kind=content.get("kind", "").lower(), format=format),
326 "thumbnail": settings.BACKUP_VIDEO_SOURCE.format(youtube_id=dubbed_id, video_format="png"),
327 }
328 else:
329 161.6 MiB 0.0 MiB content["available"] = False
330 else:
331 content["available"] = False
332 else:
333 content["available"] = False
334
335 # Get list of subtitle language codes currently available
336 161.6 MiB 0.0 MiB subtitle_lang_codes = subtitle_langs.get("{id}.srt".format(id=content.get("id")), [])
337
338 # Generate subtitle URLs for any subtitles that do exist for this content item
339 161.6 MiB 0.0 MiB subtitle_urls = [{
340 "code": lc,
341 "url": settings.STATIC_URL + "srt/{code}/subtitles/{id}.srt".format(code=lc, id=content.get("id")),
342 "name": i18n.get_language_name(lc)
343 161.6 MiB 0.0 MiB } for lc in subtitle_lang_codes]
344
345 # Sort all subtitle URLs by language code
346 161.6 MiB 0.0 MiB content["subtitle_urls"] = sorted(subtitle_urls, key=lambda x: x.get("code", ""))
347
348 161.6 MiB 0.0 MiB with i18n.translate_block(content_lang):
349 161.6 MiB 0.0 MiB content["selected_language"] = content_lang
350 161.6 MiB 0.0 MiB content["title"] = _(content["title"])
351 161.6 MiB 0.0 MiB content["description"] = _(content.get("description")) if content.get("description") else ""
352
353 161.6 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
354 try:
355 with open(CONTENT_FILEPATH + "_" + language + ".cache", "w") as f:
356 json.dump(CONTENT[language], f)
357 except IOError as e:
358 logging.warn("Annotated content cache file failed in saving with error {e}".format(e=e))
359
360 161.6 MiB 0.0 MiB return CONTENT[language]
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
346 161.6 MiB 0.0 MiB content["subtitle_urls"] = sorted(subtitle_urls, key=lambda x: x.get("code", ""))
INFO:kalite:Preloading topic tree data for language en.
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
263 216.2 MiB 0.0 MiB @profile
264 def get_content_cache(force=False, annotate=False, language=None):
265
266 216.2 MiB 0.0 MiB if not language:
267 language = settings.LANGUAGE_CODE
268
269 global CONTENT, CONTENT_FILEPATH
270
271 216.2 MiB 0.0 MiB if CONTENT is None:
272 CONTENT = {}
273 216.2 MiB 0.0 MiB if CONTENT.get(language) is None:
274 CONTENT[language] = softload_json(CONTENT_FILEPATH, logger=logging.debug, raises=False)
275 annotate = True
276
277 216.2 MiB 0.0 MiB if annotate:
278 if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP and not force:
279 content = softload_json(CONTENT_FILEPATH + "_" + language + ".cache", logger=logging.debug, raises=False)
280 if content:
281 CONTENT[language] = content
282 return CONTENT[language]
283
284 # Loop through all content items and put thumbnail urls, content urls,
285 # and subtitle urls on the content dictionary, and list all languages
286 # that the content is available in.
287 try:
288 contents_folder = os.listdir(settings.CONTENT_ROOT)
289 except OSError:
290 contents_folder = []
291
292 subtitle_langs = {}
293
294 if os.path.exists(i18n.get_srt_path()):
295 for (dirpath, dirnames, filenames) in os.walk(i18n.get_srt_path()):
296 # Only both looking at files that are inside a 'subtitles' directory
297 if os.path.basename(dirpath) == "subtitles":
298 lc = os.path.basename(os.path.dirname(dirpath))
299 for filename in filenames:
300 if filename in subtitle_langs:
301 subtitle_langs[filename].append(lc)
302 else:
303 subtitle_langs[filename] = [lc]
304
305 for content in CONTENT[language].values():
306 default_thumbnail = create_thumbnail_url(content.get("id"))
307 dubmap = i18n.get_id2oklang_map(content.get("id"))
308 if dubmap:
309 content_lang = i18n.select_best_available_language(language, available_codes=dubmap.keys()) or ""
310 if content_lang:
311 dubbed_id = dubmap.get(content_lang)
312 format = content.get("format", "")
313 if (dubbed_id + "." + format) in contents_folder:
314 content["available"] = True
315 thumbnail = create_thumbnail_url(dubbed_id) or default_thumbnail
316 content["content_urls"] = {
317 "stream": settings.CONTENT_URL + dubmap.get(content_lang) + "." + format,
318 "stream_type": "{kind}/{format}".format(kind=content.get("kind", "").lower(), format=format),
319 "thumbnail": thumbnail,
320 }
321 elif settings.BACKUP_VIDEO_SOURCE:
322 content["available"] = True
323 content["content_urls"] = {
324 "stream": settings.BACKUP_VIDEO_SOURCE.format(youtube_id=dubbed_id, video_format=format),
325 "stream_type": "{kind}/{format}".format(kind=content.get("kind", "").lower(), format=format),
326 "thumbnail": settings.BACKUP_VIDEO_SOURCE.format(youtube_id=dubbed_id, video_format="png"),
327 }
328 else:
329 content["available"] = False
330 else:
331 content["available"] = False
332 else:
333 content["available"] = False
334
335 # Get list of subtitle language codes currently available
336 subtitle_lang_codes = subtitle_langs.get("{id}.srt".format(id=content.get("id")), [])
337
338 # Generate subtitle URLs for any subtitles that do exist for this content item
339 subtitle_urls = [{
340 "code": lc,
341 "url": settings.STATIC_URL + "srt/{code}/subtitles/{id}.srt".format(code=lc, id=content.get("id")),
342 "name": i18n.get_language_name(lc)
343 } for lc in subtitle_lang_codes]
344
345 # Sort all subtitle URLs by language code
346 content["subtitle_urls"] = sorted(subtitle_urls, key=lambda x: x.get("code", ""))
347
348 with i18n.translate_block(content_lang):
349 content["selected_language"] = content_lang
350 content["title"] = _(content["title"])
351 content["description"] = _(content.get("description")) if content.get("description") else ""
352
353 if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
354 try:
355 with open(CONTENT_FILEPATH + "_" + language + ".cache", "w") as f:
356 json.dump(CONTENT[language], f)
357 except IOError as e:
358 logging.warn("Annotated content cache file failed in saving with error {e}".format(e=e))
359
360 216.2 MiB 0.0 MiB return CONTENT[language]
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
166 216.2 MiB 0.0 MiB @profile
167 def get_exercise_cache(force=False, language=None):
168
169 216.2 MiB 0.0 MiB if not language:
170 language = settings.LANGUAGE_CODE
171
172 global EXERCISES, EXERCISES_FILEPATH
173 216.2 MiB 0.0 MiB if EXERCISES is None:
174 EXERCISES = {}
175 216.2 MiB 0.0 MiB if EXERCISES.get(language) is None:
176 if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP and not force:
177 exercises = softload_json(EXERCISES_FILEPATH + "_" + language + ".cache", logger=logging.debug, raises=False)
178 if exercises:
179 EXERCISES[language] = exercises
180 return EXERCISES[language]
181 EXERCISES[language] = softload_json(EXERCISES_FILEPATH, logger=logging.debug, raises=False)
182 if language == "en": # English-language exercises live in application space, translations in user space
183 exercise_root = os.path.join(settings.KHAN_EXERCISES_DIRPATH, "exercises")
184 else:
185 exercise_root = os.path.join(settings.USER_DATA_ROOT, "exercises")
186 if os.path.exists(exercise_root):
187 exercise_path = os.path.join(exercise_root, language) if language != "en" else exercise_root
188 try:
189 exercise_templates = os.listdir(exercise_path)
190 except OSError:
191 exercise_templates = []
192 else:
193 exercise_templates = []
194
195 for exercise in EXERCISES[language].values():
196 exercise_file = exercise["name"] + ".html"
197 exercise_template = exercise_file
198 exercise_lang = "en"
199
200 if exercise.get("uses_assessment_items", False):
201 available = False
202 items = []
203 for item in exercise.get("all_assessment_items","[]"):
204 item = json.loads(item)
205 if get_assessment_item_data(request=None, assessment_item_id=item.get("id")):
206 items.append(item)
207 available = True
208 exercise["all_assessment_items"] = items
209 else:
210 available = exercise_template in exercise_templates
211
212 # Get the language codes for exercise templates that exist
213 # Try to minimize the number of os.path.exists calls (since they're a bottleneck) by using the same
214 # precedence rules in i18n.select_best_available_languages
215 available_langs = set(["en"] + [language]*available)
216 # Return the best available exercise template
217 exercise_lang = i18n.select_best_available_language(language, available_codes=available_langs)
218
219 if exercise_lang == "en":
220 exercise_template = exercise_file
221 else:
222 exercise_template = os.path.join(exercise_lang, exercise_file)
223
224 with i18n.translate_block(exercise_lang):
225 exercise["available"] = available
226 exercise["lang"] = exercise_lang
227 exercise["template"] = exercise_template
228 exercise["title"] = _(exercise.get("title", ""))
229 exercise["description"] = _(exercise.get("description", "")) if exercise.get("description") else ""
230
231 if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
232 try:
233 with open(EXERCISES_FILEPATH + "_" + language + ".cache", "w") as f:
234 json.dump(EXERCISES[language], f)
235 except IOError as e:
236 logging.warn("Annotated exercise cache file failed in saving with error {e}".format(e=e))
237
238 216.2 MiB 0.0 MiB return EXERCISES[language]
Raw
gistfile5.txt
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
166 215.7 MiB 0.0 MiB @profile
167 def get_exercise_cache(force=False, language=None):
168
169 215.7 MiB 0.0 MiB if not language:
170 language = settings.LANGUAGE_CODE
171
172 global EXERCISES, EXERCISES_FILEPATH
173 215.7 MiB 0.0 MiB if EXERCISES is None:
174 EXERCISES = {}
175 215.7 MiB 0.0 MiB if EXERCISES.get(language) is None:
176 215.7 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP and not force:
177 exercises = softload_json(EXERCISES_FILEPATH + "_" + language + ".cache", logger=logging.debug, raises=False)
178 if exercises:
179 EXERCISES[language] = exercises
180 return EXERCISES[language]
181 219.6 MiB 3.9 MiB EXERCISES[language] = softload_json(EXERCISES_FILEPATH, logger=logging.debug, raises=False)
182 219.6 MiB 0.0 MiB if language == "en": # English-language exercises live in application space, translations in user space
183 exercise_root = os.path.join(settings.KHAN_EXERCISES_DIRPATH, "exercises")
184 else:
185 219.6 MiB 0.0 MiB exercise_root = os.path.join(settings.USER_DATA_ROOT, "exercises")
186 219.6 MiB 0.0 MiB if os.path.exists(exercise_root):
187 219.6 MiB 0.0 MiB exercise_path = os.path.join(exercise_root, language) if language != "en" else exercise_root
188 219.6 MiB 0.0 MiB try:
189 219.6 MiB 0.0 MiB exercise_templates = os.listdir(exercise_path)
190 219.6 MiB 0.0 MiB except OSError:
191 219.6 MiB 0.0 MiB exercise_templates = []
192 else:
193 exercise_templates = []
194
195 239.6 MiB 20.0 MiB for exercise in EXERCISES[language].values():
196 239.6 MiB -0.0 MiB exercise_file = exercise["name"] + ".html"
197 239.6 MiB 0.0 MiB exercise_template = exercise_file
198 239.6 MiB 0.0 MiB exercise_lang = "en"
199
200 239.6 MiB 0.0 MiB if exercise.get("uses_assessment_items", False):
201 239.6 MiB 0.0 MiB available = False
202 239.6 MiB 0.0 MiB items = []
203 239.6 MiB 0.0 MiB for item in exercise.get("all_assessment_items","[]"):
204 239.6 MiB -0.0 MiB item = json.loads(item)
205 239.6 MiB 0.0 MiB if get_assessment_item_data(request=None, assessment_item_id=item.get("id")):
206 items.append(item)
207 available = True
208 239.6 MiB 0.0 MiB exercise["all_assessment_items"] = items
209 else:
210 239.5 MiB -0.1 MiB available = exercise_template in exercise_templates
211
212 # Get the language codes for exercise templates that exist
213 # Try to minimize the number of os.path.exists calls (since they're a bottleneck) by using the same
214 # precedence rules in i18n.select_best_available_languages
215 239.5 MiB 0.0 MiB available_langs = set(["en"] + [language]*available)
216 # Return the best available exercise template
217 239.5 MiB 0.0 MiB exercise_lang = i18n.select_best_available_language(language, available_codes=available_langs)
218
219 239.6 MiB 0.1 MiB if exercise_lang == "en":
220 239.6 MiB 0.0 MiB exercise_template = exercise_file
221 else:
222 exercise_template = os.path.join(exercise_lang, exercise_file)
223
224 239.6 MiB 0.0 MiB with i18n.translate_block(exercise_lang):
225 239.6 MiB 0.0 MiB exercise["available"] = available
226 239.6 MiB 0.0 MiB exercise["lang"] = exercise_lang
227 239.6 MiB 0.0 MiB exercise["template"] = exercise_template
228 239.6 MiB 0.0 MiB exercise["title"] = _(exercise.get("title", ""))
229 239.6 MiB 0.0 MiB exercise["description"] = _(exercise.get("description", "")) if exercise.get("description") else ""
230
231 239.6 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
232 try:
233 with open(EXERCISES_FILEPATH + "_" + language + ".cache", "w") as f:
234 json.dump(EXERCISES[language], f)
235 except IOError as e:
236 logging.warn("Annotated exercise cache file failed in saving with error {e}".format(e=e))
237
238 239.6 MiB 0.0 MiB return EXERCISES[language]
INFO:kalite:Preloading content data for language pt-BR.
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
263 239.6 MiB 0.0 MiB @profile
264 def get_content_cache(force=False, annotate=False, language=None):
265
266 239.6 MiB 0.0 MiB if not language:
267 language = settings.LANGUAGE_CODE
268
269 global CONTENT, CONTENT_FILEPATH
270
271 239.6 MiB 0.0 MiB if CONTENT is None:
272 CONTENT = {}
273 239.6 MiB 0.0 MiB if CONTENT.get(language) is None:
274 290.6 MiB 51.0 MiB CONTENT[language] = softload_json(CONTENT_FILEPATH, logger=logging.debug, raises=False)
275 290.6 MiB 0.0 MiB annotate = True
276
277 290.6 MiB 0.0 MiB if annotate:
278 290.6 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP and not force:
279 content = softload_json(CONTENT_FILEPATH + "_" + language + ".cache", logger=logging.debug, raises=False)
280 if content:
281 CONTENT[language] = content
282 return CONTENT[language]
283
284 # Loop through all content items and put thumbnail urls, content urls,
285 # and subtitle urls on the content dictionary, and list all languages
286 # that the content is available in.
287 290.6 MiB 0.0 MiB try:
288 290.6 MiB 0.0 MiB contents_folder = os.listdir(settings.CONTENT_ROOT)
289 except OSError:
290 contents_folder = []
291
292 290.6 MiB 0.0 MiB subtitle_langs = {}
293
294 290.6 MiB 0.0 MiB if os.path.exists(i18n.get_srt_path()):
295 290.6 MiB 0.0 MiB for (dirpath, dirnames, filenames) in os.walk(i18n.get_srt_path()):
296 # Only both looking at files that are inside a 'subtitles' directory
297 290.6 MiB 0.0 MiB if os.path.basename(dirpath) == "subtitles":
298 290.6 MiB 0.0 MiB lc = os.path.basename(os.path.dirname(dirpath))
299 290.6 MiB 0.0 MiB for filename in filenames:
300 290.6 MiB 0.0 MiB if filename in subtitle_langs:
301 290.6 MiB 0.0 MiB subtitle_langs[filename].append(lc)
302 else:
303 290.6 MiB 0.0 MiB subtitle_langs[filename] = [lc]
304
305 290.6 MiB 0.0 MiB for content in CONTENT[language].values():
306 290.6 MiB 0.0 MiB default_thumbnail = create_thumbnail_url(content.get("id"))
307 290.6 MiB 0.0 MiB dubmap = i18n.get_id2oklang_map(content.get("id"))
308 290.6 MiB 0.0 MiB if dubmap:
309 290.6 MiB 0.0 MiB content_lang = i18n.select_best_available_language(language, available_codes=dubmap.keys()) or ""
310 290.6 MiB 0.0 MiB if content_lang:
311 290.6 MiB 0.0 MiB dubbed_id = dubmap.get(content_lang)
312 290.6 MiB 0.0 MiB format = content.get("format", "")
313 290.6 MiB 0.0 MiB if (dubbed_id + "." + format) in contents_folder:
314 290.6 MiB 0.0 MiB content["available"] = True
315 290.6 MiB 0.0 MiB thumbnail = create_thumbnail_url(dubbed_id) or default_thumbnail
316 290.6 MiB 0.0 MiB content["content_urls"] = {
317 290.6 MiB 0.0 MiB "stream": settings.CONTENT_URL + dubmap.get(content_lang) + "." + format,
318 290.6 MiB 0.0 MiB "stream_type": "{kind}/{format}".format(kind=content.get("kind", "").lower(), format=format),
319 290.6 MiB 0.0 MiB "thumbnail": thumbnail,
320 }
321 290.6 MiB 0.0 MiB elif settings.BACKUP_VIDEO_SOURCE:
322 content["available"] = True
323 content["content_urls"] = {
324 "stream": settings.BACKUP_VIDEO_SOURCE.format(youtube_id=dubbed_id, video_format=format),
325 "stream_type": "{kind}/{format}".format(kind=content.get("kind", "").lower(), format=format),
326 "thumbnail": settings.BACKUP_VIDEO_SOURCE.format(youtube_id=dubbed_id, video_format="png"),
327 }
328 else:
329 290.6 MiB 0.0 MiB content["available"] = False
330 else:
331 content["available"] = False
332 else:
333 content["available"] = False
334
335 # Get list of subtitle language codes currently available
336 290.6 MiB 0.0 MiB subtitle_lang_codes = subtitle_langs.get("{id}.srt".format(id=content.get("id")), [])
337
338 # Generate subtitle URLs for any subtitles that do exist for this content item
339 290.6 MiB 0.0 MiB subtitle_urls = [{
340 "code": lc,
341 "url": settings.STATIC_URL + "srt/{code}/subtitles/{id}.srt".format(code=lc, id=content.get("id")),
342 "name": i18n.get_language_name(lc)
343 290.6 MiB 0.0 MiB } for lc in subtitle_lang_codes]
344
345 # Sort all subtitle URLs by language code
346 290.6 MiB 0.0 MiB content["subtitle_urls"] = sorted(subtitle_urls, key=lambda x: x.get("code", ""))
347
348 290.6 MiB 0.0 MiB with i18n.translate_block(content_lang):
349 290.6 MiB 0.0 MiB content["selected_language"] = content_lang
350 290.6 MiB 0.0 MiB content["title"] = _(content["title"])
351 290.6 MiB 0.0 MiB content["description"] = _(content.get("description")) if content.get("description") else ""
352
353 290.6 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
354 try:
355 with open(CONTENT_FILEPATH + "_" + language + ".cache", "w") as f:
356 json.dump(CONTENT[language], f)
357 except IOError as e:
358 logging.warn("Annotated content cache file failed in saving with error {e}".format(e=e))
359
360 290.6 MiB 0.0 MiB return CONTENT[language]
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
346 290.6 MiB 0.0 MiB content["subtitle_urls"] = sorted(subtitle_urls, key=lambda x: x.get("code", ""))
INFO:kalite:Preloading topic tree data for language pt-BR.
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
263 336.7 MiB 0.0 MiB @profile
264 def get_content_cache(force=False, annotate=False, language=None):
265
266 336.7 MiB 0.0 MiB if not language:
267 language = settings.LANGUAGE_CODE
268
269 global CONTENT, CONTENT_FILEPATH
270
271 336.7 MiB 0.0 MiB if CONTENT is None:
272 CONTENT = {}
273 336.7 MiB 0.0 MiB if CONTENT.get(language) is None:
274 376.2 MiB 39.5 MiB CONTENT[language] = softload_json(CONTENT_FILEPATH, logger=logging.debug, raises=False)
275 376.2 MiB 0.0 MiB annotate = True
276
277 376.2 MiB 0.0 MiB if annotate:
278 376.2 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP and not force:
279 content = softload_json(CONTENT_FILEPATH + "_" + language + ".cache", logger=logging.debug, raises=False)
280 if content:
281 CONTENT[language] = content
282 return CONTENT[language]
283
284 # Loop through all content items and put thumbnail urls, content urls,
285 # and subtitle urls on the content dictionary, and list all languages
286 # that the content is available in.
287 376.2 MiB 0.0 MiB try:
288 376.2 MiB 0.0 MiB contents_folder = os.listdir(settings.CONTENT_ROOT)
289 except OSError:
290 contents_folder = []
291
292 376.2 MiB 0.0 MiB subtitle_langs = {}
293
294 376.2 MiB 0.0 MiB if os.path.exists(i18n.get_srt_path()):
295 376.2 MiB 0.0 MiB for (dirpath, dirnames, filenames) in os.walk(i18n.get_srt_path()):
296 # Only both looking at files that are inside a 'subtitles' directory
297 376.2 MiB 0.0 MiB if os.path.basename(dirpath) == "subtitles":
298 376.2 MiB 0.0 MiB lc = os.path.basename(os.path.dirname(dirpath))
299 376.2 MiB 0.0 MiB for filename in filenames:
300 376.2 MiB 0.0 MiB if filename in subtitle_langs:
301 376.2 MiB 0.0 MiB subtitle_langs[filename].append(lc)
302 else:
303 376.2 MiB 0.0 MiB subtitle_langs[filename] = [lc]
304
305 376.2 MiB 0.0 MiB for content in CONTENT[language].values():
306 376.2 MiB 0.0 MiB default_thumbnail = create_thumbnail_url(content.get("id"))
307 376.2 MiB 0.0 MiB dubmap = i18n.get_id2oklang_map(content.get("id"))
308 376.2 MiB 0.0 MiB if dubmap:
309 376.2 MiB 0.0 MiB content_lang = i18n.select_best_available_language(language, available_codes=dubmap.keys()) or ""
310 376.2 MiB 0.0 MiB if content_lang:
311 376.2 MiB 0.0 MiB dubbed_id = dubmap.get(content_lang)
312 376.2 MiB 0.0 MiB format = content.get("format", "")
313 376.2 MiB 0.0 MiB if (dubbed_id + "." + format) in contents_folder:
314 376.2 MiB 0.0 MiB content["available"] = True
315 376.2 MiB 0.0 MiB thumbnail = create_thumbnail_url(dubbed_id) or default_thumbnail
316 376.2 MiB 0.0 MiB content["content_urls"] = {
317 376.2 MiB 0.0 MiB "stream": settings.CONTENT_URL + dubmap.get(content_lang) + "." + format,
318 376.2 MiB 0.0 MiB "stream_type": "{kind}/{format}".format(kind=content.get("kind", "").lower(), format=format),
319 376.2 MiB 0.0 MiB "thumbnail": thumbnail,
320 }
321 376.2 MiB 0.0 MiB elif settings.BACKUP_VIDEO_SOURCE:
322 content["available"] = True
323 content["content_urls"] = {
324 "stream": settings.BACKUP_VIDEO_SOURCE.format(youtube_id=dubbed_id, video_format=format),
325 "stream_type": "{kind}/{format}".format(kind=content.get("kind", "").lower(), format=format),
326 "thumbnail": settings.BACKUP_VIDEO_SOURCE.format(youtube_id=dubbed_id, video_format="png"),
327 }
328 else:
329 376.2 MiB 0.0 MiB content["available"] = False
330 else:
331 content["available"] = False
332 else:
333 content["available"] = False
334
335 # Get list of subtitle language codes currently available
336 376.2 MiB 0.0 MiB subtitle_lang_codes = subtitle_langs.get("{id}.srt".format(id=content.get("id")), [])
337
338 # Generate subtitle URLs for any subtitles that do exist for this content item
339 376.2 MiB 0.0 MiB subtitle_urls = [{
340 "code": lc,
341 "url": settings.STATIC_URL + "srt/{code}/subtitles/{id}.srt".format(code=lc, id=content.get("id")),
342 "name": i18n.get_language_name(lc)
343 376.2 MiB 0.0 MiB } for lc in subtitle_lang_codes]
344
345 # Sort all subtitle URLs by language code
346 376.2 MiB 0.0 MiB content["subtitle_urls"] = sorted(subtitle_urls, key=lambda x: x.get("code", ""))
347
348 376.2 MiB 0.0 MiB with i18n.translate_block(content_lang):
349 376.2 MiB 0.0 MiB content["selected_language"] = content_lang
350 376.2 MiB 0.0 MiB content["title"] = _(content["title"])
351 376.2 MiB 0.0 MiB content["description"] = _(content.get("description")) if content.get("description") else ""
352
353 376.2 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
354 try:
355 with open(CONTENT_FILEPATH + "_" + language + ".cache", "w") as f:
356 json.dump(CONTENT[language], f)
357 except IOError as e:
358 logging.warn("Annotated content cache file failed in saving with error {e}".format(e=e))
359
360 376.2 MiB 0.0 MiB return CONTENT[language]
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
346 376.2 MiB 0.0 MiB content["subtitle_urls"] = sorted(subtitle_urls, key=lambda x: x.get("code", ""))
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
166 376.2 MiB 0.0 MiB @profile
167 def get_exercise_cache(force=False, language=None):
168
169 376.2 MiB 0.0 MiB if not language:
170 language = settings.LANGUAGE_CODE
171
172 global EXERCISES, EXERCISES_FILEPATH
173 376.2 MiB 0.0 MiB if EXERCISES is None:
174 EXERCISES = {}
175 376.2 MiB 0.0 MiB if EXERCISES.get(language) is None:
176 376.2 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP and not force:
177 exercises = softload_json(EXERCISES_FILEPATH + "_" + language + ".cache", logger=logging.debug, raises=False)
178 if exercises:
179 EXERCISES[language] = exercises
180 return EXERCISES[language]
181 385.2 MiB 9.0 MiB EXERCISES[language] = softload_json(EXERCISES_FILEPATH, logger=logging.debug, raises=False)
182 385.2 MiB 0.0 MiB if language == "en": # English-language exercises live in application space, translations in user space
183 exercise_root = os.path.join(settings.KHAN_EXERCISES_DIRPATH, "exercises")
184 else:
185 385.2 MiB 0.0 MiB exercise_root = os.path.join(settings.USER_DATA_ROOT, "exercises")
186 385.2 MiB 0.0 MiB if os.path.exists(exercise_root):
187 385.2 MiB 0.0 MiB exercise_path = os.path.join(exercise_root, language) if language != "en" else exercise_root
188 385.2 MiB 0.0 MiB try:
189 385.2 MiB 0.0 MiB exercise_templates = os.listdir(exercise_path)
190 385.2 MiB 0.0 MiB except OSError:
191 385.2 MiB 0.0 MiB exercise_templates = []
192 else:
193 exercise_templates = []
194
195 411.7 MiB 26.5 MiB for exercise in EXERCISES[language].values():
196 411.7 MiB -0.0 MiB exercise_file = exercise["name"] + ".html"
197 411.7 MiB 0.0 MiB exercise_template = exercise_file
198 411.7 MiB 0.0 MiB exercise_lang = "en"
199
200 411.7 MiB 0.0 MiB if exercise.get("uses_assessment_items", False):
201 411.7 MiB 0.0 MiB available = False
202 411.7 MiB 0.0 MiB items = []
203 411.7 MiB 0.0 MiB for item in exercise.get("all_assessment_items","[]"):
204 411.7 MiB 0.0 MiB item = json.loads(item)
205 411.7 MiB 0.0 MiB if get_assessment_item_data(request=None, assessment_item_id=item.get("id")):
206 items.append(item)
207 available = True
208 411.7 MiB 0.0 MiB exercise["all_assessment_items"] = items
209 else:
210 411.6 MiB -0.1 MiB available = exercise_template in exercise_templates
211
212 # Get the language codes for exercise templates that exist
213 # Try to minimize the number of os.path.exists calls (since they're a bottleneck) by using the same
214 # precedence rules in i18n.select_best_available_languages
215 411.6 MiB 0.0 MiB available_langs = set(["en"] + [language]*available)
216 # Return the best available exercise template
217 411.6 MiB 0.0 MiB exercise_lang = i18n.select_best_available_language(language, available_codes=available_langs)
218
219 411.7 MiB 0.1 MiB if exercise_lang == "en":
220 411.7 MiB 0.0 MiB exercise_template = exercise_file
221 else:
222 exercise_template = os.path.join(exercise_lang, exercise_file)
223
224 411.7 MiB 0.0 MiB with i18n.translate_block(exercise_lang):
225 411.7 MiB 0.0 MiB exercise["available"] = available
226 411.7 MiB 0.0 MiB exercise["lang"] = exercise_lang
227 411.7 MiB 0.0 MiB exercise["template"] = exercise_template
228 411.7 MiB 0.0 MiB exercise["title"] = _(exercise.get("title", ""))
229 411.7 MiB 0.0 MiB exercise["description"] = _(exercise.get("description", "")) if exercise.get("description") else ""
230
231 411.7 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
232 try:
233 with open(EXERCISES_FILEPATH + "_" + language + ".cache", "w") as f:
234 json.dump(EXERCISES[language], f)
235 except IOError as e:
236 logging.warn("Annotated exercise cache file failed in saving with error {e}".format(e=e))
237
238 411.7 MiB 0.0 MiB return EXERCISES[language]
Filename: /home/aron/src/github.com/aronasorman/ka-lite/kalite/topic_tools/__init__.py
Line # Mem usage Increment Line Contents
================================================
56 290.6 MiB 0.0 MiB @profile
57 def get_topic_tree(force=False, annotate=False, channel=None, language=None, parent=None):
58
59 # Hardcode the Brazilian Portuguese mapping that only the central server knows about
60 # TODO(jamalex): BURN IT ALL DOWN!
61 290.6 MiB 0.0 MiB if language == "pt-BR":
62 290.6 MiB 0.0 MiB language = "pt"
63
64 290.6 MiB 0.0 MiB if not channel:
65 290.6 MiB 0.0 MiB channel = settings.CHANNEL
66
67 290.6 MiB 0.0 MiB if not language:
68 language = settings.LANGUAGE_CODE
69
70 global TOPICS, TOPICS_FILEPATHS
71 290.6 MiB 0.0 MiB if not TOPICS:
72 TOPICS = {}
73 290.6 MiB 0.0 MiB if TOPICS.get(channel) is None:
74 TOPICS[channel] = {}
75 290.6 MiB 0.0 MiB if annotate or TOPICS.get(channel, {}).get(language) is None:
76 336.7 MiB 46.1 MiB topics = softload_json(TOPICS_FILEPATHS.get(channel), logger=logging.debug, raises=False)
77
78 # Just loaded from disk, so have to restamp.
79 336.7 MiB 0.0 MiB annotate = True
80
81 336.7 MiB 0.0 MiB if annotate:
82 336.7 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP and not force:
83 topics = softload_json(TOPICS_FILEPATHS.get(channel) + "_" + language + ".cache", logger=logging.debug, raises=False)
84 if topics:
85 TOPICS[channel][language] = topics
86 return TOPICS[channel][language]
87
88 336.7 MiB 0.0 MiB flat_topic_tree = []
89
90 # Loop through all the nodes in the topic tree
91 # and cross reference with the content_cache to check availability.
92 376.2 MiB 39.5 MiB content_cache = get_content_cache(language=language)
93 411.7 MiB 35.5 MiB exercise_cache = get_exercise_cache(language=language)
94
95 411.7 MiB 0.0 MiB def recurse_nodes(node, parent=""):
96
97 node["parent"] = parent
98
99 node.pop("child_data", None)
100
101 child_availability = []
102
103 child_ids = [child.get("id") for child in node.get("children", [])]
104
105 # Do the recursion
106 for child in node.get("children", []):
107 recurse_nodes(child, node.get("id"))
108 child_availability.append(child.get("available", False))
109
110 if child_ids:
111 node["children"] = child_ids
112
113 # If child_availability is empty then node has no children so we can determine availability
114 if child_availability:
115 node["available"] = any(child_availability)
116 else:
117 # By default this is very charitable, assuming if something has not been annotated
118 # it is available.
119 if node.get("kind") == "Exercise":
120 cache_node = exercise_cache.get(node.get("id"), {})
121 else:
122 cache_node = content_cache.get(node.get("id"), {})
123 node["available"] = cache_node.get("available", True)
124
125 # Translate everything for good measure
126 with i18n.translate_block(language):
127 node["title"] = _(node.get("title", ""))
128 node["description"] = _(node.get("description", "")) if node.get("description") else ""
129
130 411.8 MiB 0.1 MiB flat_topic_tree.append(node)
131
132 recurse_nodes(topics)
133
134 411.8 MiB 0.0 MiB TOPICS[channel][language] = flat_topic_tree
135
136 411.8 MiB 0.0 MiB if settings.DO_NOT_RELOAD_CONTENT_CACHE_AT_STARTUP:
137 try:
138 with open(TOPICS_FILEPATHS.get(channel) + "_" + language + ".cache", "w") as f:
139 json.dump(TOPICS[channel][language], f)
140 except IOError as e:
141 logging.warn("Annotated topic cache file failed in saving with error {e}".format(e=e))
142
143 411.8 MiB 0.0 MiB if parent:
144 return filter(lambda x: x.get("parent") == parent, TOPICS[channel][language])
145 else:
146 411.8 MiB 0.0 MiB return TOPICS[channel][language]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment