terapyon/get_course.py

## get_course.py
import json
import gzip
from operator import itemgetter
from pathlib import Path
import asyncio


def get_logfiles(path):
    for p in path.glob("*.gz"):
        date_str = p.stem.split('-')[1]
        yield p, int(date_str)


async def get_log_lines(p):
    with gzip.open(p, 'r') as f:
        for line in f:
            if not isinstance(line, str):
                line = line.decode('utf-8')
            # await asyncio.sleep(1)
            yield line


def check_course_id(course_id, line):
    j = json.loads(line)
    return course_id == j.get('context', {}).get('course_id')


async def a_get_sorted_files(sorted_files):
    for f in sorted_files:
        yield f


async def get_line_for_course(sorted_files, course_id, s_date, e_date):
    async for p, date_int in a_get_sorted_files(sorted_files):
        # await asyncio.sleep(1)
        if date_int >= s_date and date_int <= e_date:
            async for line in get_log_lines(p):
                if check_course_id(course_id, line):
                    yield line


def get_sorted_files(path="logs"):
    path = Path(path)
    sorted_files = sorted(get_logfiles(path), key=itemgetter(1))
    return sorted_files


## main.py

from aiohttp import web

import get_course


async def index(request: web.Request) -> web.Response:
    return web.Response(
        content_type="text/html",
        body="""<html>
  <body>
    <h1>Open edX course log Service</h1>
    <form action="/get-data/" method="post">
      <label>
        Course ID: <input type="text" name="course_id" />
      </label>
      <label>
        Start Date by int (20180821): <input type="text" name="s_date" />
      </label>
      <label>
        End Date by int (20180822): <input type="text" name="e_date" />
      </label>
      <button>Submit</button>
    </form>
  </body>
</html>
""",
    )


async def get_data(request: web.Request) -> web.StreamResponse:
    data = await request.post()
    course_id = data.get("course_id")
    s_date = int(data.get("s_date", 0))
    e_date = int(data.get("e_date", 0))
    # print("C", course_id)

    sorted_files = get_course.get_sorted_files("/Users/terapyon/tmp/koala/logs")
    obj = get_course.get_line_for_course(sorted_files, course_id, s_date, e_date)

    response = web.StreamResponse()
    await response.prepare(request)
    # async with obj as stream:
    async for data in obj:
        await response.write(data.encode('utf-8'))
    await response.write_eof()
    return response


def build_app() -> web.Application:
    app = web.Application()
    app.add_routes([web.get("/", index)])
    app.add_routes([web.post("/get-data/", get_data)])
    return app


if __name__ == "__main__":
    web.run_app(build_app())
	import json
	import gzip
	from operator import itemgetter
	from pathlib import Path
	import asyncio


	def get_logfiles(path):
	for p in path.glob("*.gz"):
	date_str = p.stem.split('-')[1]
	yield p, int(date_str)


	async def get_log_lines(p):
	with gzip.open(p, 'r') as f:
	for line in f:
	if not isinstance(line, str):
	line = line.decode('utf-8')
	# await asyncio.sleep(1)
	yield line


	def check_course_id(course_id, line):
	j = json.loads(line)
	return course_id == j.get('context', {}).get('course_id')


	async def a_get_sorted_files(sorted_files):
	for f in sorted_files:
	yield f


	async def get_line_for_course(sorted_files, course_id, s_date, e_date):
	async for p, date_int in a_get_sorted_files(sorted_files):
	# await asyncio.sleep(1)
	if date_int >= s_date and date_int <= e_date:
	async for line in get_log_lines(p):
	if check_course_id(course_id, line):
	yield line


	def get_sorted_files(path="logs"):
	path = Path(path)
	sorted_files = sorted(get_logfiles(path), key=itemgetter(1))
	return sorted_files

	from aiohttp import web

	import get_course


	async def index(request: web.Request) -> web.Response:
	return web.Response(
	content_type="text/html",
	body="""<html>
	<body>
	<h1>Open edX course log Service</h1>
	<form action="/get-data/" method="post">
	<label>
	Course ID: <input type="text" name="course_id" />
	</label>
	<label>
	Start Date by int (20180821): <input type="text" name="s_date" />
	</label>
	<label>
	End Date by int (20180822): <input type="text" name="e_date" />
	</label>
	<button>Submit</button>
	</form>
	</body>
	</html>
	""",
	)


	async def get_data(request: web.Request) -> web.StreamResponse:
	data = await request.post()
	course_id = data.get("course_id")
	s_date = int(data.get("s_date", 0))
	e_date = int(data.get("e_date", 0))
	# print("C", course_id)

	sorted_files = get_course.get_sorted_files("/Users/terapyon/tmp/koala/logs")
	obj = get_course.get_line_for_course(sorted_files, course_id, s_date, e_date)

	response = web.StreamResponse()
	await response.prepare(request)
	# async with obj as stream:
	async for data in obj:
	await response.write(data.encode('utf-8'))
	await response.write_eof()
	return response


	def build_app() -> web.Application:
	app = web.Application()
	app.add_routes([web.get("/", index)])
	app.add_routes([web.post("/get-data/", get_data)])
	return app


	if __name__ == "__main__":
	web.run_app(build_app())