Class.ts

import puppeteer, { Browser, Page } from 'puppeteer';
import { LearningAppImportCredentials } from './sdk/learningAppImportStartup';
import { AppCourse, AppLevel, AppTopic, LevelMasteryRecord } from './sdk/learningAppImport';
import * as cheerio from 'cheerio';

interface NoredinkCourse {
  id: number;
  name: string;
}

interface NoredinkTopicsAndLevelsJSONCourse {
  id: number;
  grade_indices: number[];
}

interface NoredinkTopicsAndLevelsJSON {
  pageState: {
    assignments: NoredinkTopic[];
  };
  courses: NoredinkTopicsAndLevelsJSONCourse[];
}

interface NoredinkWritingStudentGrade {
  id: number;
  studentStateId: number;
  grade: number;
}

interface WritingMasteryLevelsJSON {
  students: NoredinkWritingStudentGrade[];
  statesById: [id: number, state: string][];
}

interface NoredinkLevel {
  id: number;
  name: string;
  passage: boolean;
}

interface NoredinkTopic {
  id: number;
  name: string;
  type: string;
  topics: NoredinkLevel[];
  gradeLevels: number[];
}

interface NoredinkStudent {
  id: number;
  topics: NoredinkStudentTopic[];
}

interface NoredinkStudentTopic {
  id: number;
  questionsAnswered: number;
  questionsAnsweredCorrectly: number;
  completion: number;
}

export class NoredinkEntitiesMapper {
  credentials: LearningAppImportCredentials;

  constructor(credentials: LearningAppImportCredentials) {
    this.credentials = credentials;
  }

  mapNoredinkCourseToAppCourse(course: NoredinkCourse): AppCourse {
    return {
      credentialId: this.credentials.id,
      courseName: course.name,
      courseId: `${course.id}`,
      subjectKeys: course.name.toLowerCase().includes('writing') ? 'Writing' : 'Language',
    };
  }

  mapNoredinkWritingStudentGradeToLevelMasteryRecord(
    student: NoredinkWritingStudentGrade,
    appTopic: AppTopic,
    appLevel: AppLevel,
    graded?: boolean
  ): LevelMasteryRecord {
    return graded
      ? {
          date: new Date().toISOString(),
          credentialId: this.credentials.id,
          levelId: appLevel.levelId,
          topicId: appTopic.topicId,
          studentId: `${student.id || student.studentStateId}`,
          activityUnitsAttempted: 1,
          activityUnitsCorrect: student.grade > 50 ? 1 : 0,
          masteryPercentage: student.grade,
        }
      : {
          date: new Date().toISOString(),
          credentialId: this.credentials.id,
          levelId: appLevel.levelId,
          topicId: appTopic.topicId,
          studentId: `${student.id || student.studentStateId}`,
          activityUnitsAttempted: 1,
          activityUnitsCorrect: 0,
          masteryPercentage: 0,
        };
  }

  mapNoredinkLevelToAppLevel(
    level: NoredinkTopic | NoredinkLevel,
    topicId: number,
    order: number,
    gradeLevel: number
  ): AppLevel {
    return {
      credentialId: this.credentials.id,
      levelName: level.name,
      levelId: `${level.id}`,
      gradeLevel: `${gradeLevel}`,
      levelOrder: order,
      topicId: `${topicId}`,
    };
  }

  mapNoredinkTopicToAppTopic(topic: NoredinkTopic, course: AppCourse, order: number): AppTopic {
    return {
      credentialId: this.credentials.id,
      topicName: topic.name,
      topicId: `${topic.id}`,
      topicOrder: order,
      courseId: `${course.courseId}`,
    };
  }
}

export class NoredinkAppScrapper {
  credentials: LearningAppImportCredentials;
  mapper: NoredinkEntitiesMapper;
  navigator: Page;
  browser: Browser;

  constructor(credentials: LearningAppImportCredentials) {
    this.credentials = credentials;
    this.mapper = new NoredinkEntitiesMapper(credentials);
  }

  public async init() {
    this.browser = await puppeteer.launch({ headless: false, userDataDir: './userdata' });
    this.navigator = await this.browser.newPage();
    await this.navigator.setViewport({ width: 1920, height: 955 });
    await this._login();
  }

  public async close() {
    await this.browser.close();
  }

  private async _login() {
    await this.navigator.goto('https://www.noredink.com/teach/dashboard');
    if (this.navigator.url() === 'https://www.noredink.com/teach/dashboard') return;
    const loginWithPasswordButtonSelector = '#log_in_with_password';
    await this.navigator.click(loginWithPasswordButtonSelector);
    const loginFormSelector = '.manual-login-form';
    await this.navigator.waitForSelector(loginFormSelector);
    const emailInputSelector = '#Nri-Ui-TextInput-Email-or-username';
    await this.navigator.type(emailInputSelector, this.credentials.username);
    const passwordInputSelector = '#Nri-Ui-TextInput-Password';
    await this.navigator.type(passwordInputSelector, this.credentials.password);
    const submitButtonSelector = 'button[type=submit]';
    await this.navigator.click(`${loginFormSelector} ${submitButtonSelector}`);
    await this.navigator.waitForNavigation();
    await this.navigator.screenshot({ path: 'screenshot.png' });
  }

  private async _getHTMLElementAttributeASJSON(url: string, elementId: string, attribute: string) {
    const html = await this._getHTMLOfPage(url);
    const $ = cheerio.load(html);
    return JSON.parse($(elementId).attr(attribute).replaceAll('\n', ''));
  }

  private async _getContentOfPageASJSON(url: string) {
    await this.navigator.goto(url);
    const innerText = await this.navigator.evaluate(() => {
      return JSON.parse(document.querySelector('body').innerText);
    });
    return innerText;
  }

  private async _getHTMLOfPage(url: string) {
    await new Promise(resolve => setTimeout(resolve, 1000));
    await this.navigator.goto(url);
    return this.navigator.content();
  }

  public async getCourses(): Promise<AppCourse[]> {
    const json = await this._getHTMLElementAttributeASJSON(
      'https://www.noredink.com/teach/dashboard',
      '#teacher-dashboard-elm-flags',
      'data-flags'
    );
    return json.courses.map((course: NoredinkCourse) => this.mapper.mapNoredinkCourseToAppCourse(course));
  }

  public async getTopicsAndLevels(courses: AppCourse[]) {
    const language = await this.getTopicsAndLevelsLanguage(courses);
    const writing = await this.getTopicsAndLevelsWriting(courses);
    return {
      topics: [...language.topics, ...writing.topics],
      levels: [...language.levels, ...writing.levels],
      levelMasteryRecords: [...language.levelMasteryRecords, ...language.levelMasteryRecords],
    };
  }

  private async getTopicsAndLevelsWriting(courses: AppCourse[]) {
    const appTopicsAndLevels = { topics: [], levels: [], levelMasteryRecords: [] };
    const noredinkCourses = [courses.filter(course => course.subjectKeys === 'Writing')[0]];
    for (let i = 0; i < noredinkCourses.length; i++) {
      const course = noredinkCourses[i];

      const json = (await this._getHTMLElementAttributeASJSON(
        `https://www.noredink.com/teach/courses/${course.courseId}/assignments.html`,
        '#assignments-page-elm-flags',
        'data-flags'
      )) as NoredinkTopicsAndLevelsJSON;
      const { assignments } = json.pageState;

      for (let j = 0; j < assignments.length; j++) {
        const topic = assignments[j];
        if (topic.type !== 'Guided Draft' && topic.type !== 'Quick Write') continue;
        const appTopic = this.mapper.mapNoredinkTopicToAppTopic(topic, course, j + 1);
        const gradeLevel = json.courses.find(c => `${c.id}` === course.courseId).grade_indices.sort()[0];
        const appLevel = this.mapper.mapNoredinkLevelToAppLevel(topic, topic.id, j + 1, gradeLevel);
        const levelMasteryRecords = await this.getMasteryRecordsWriting(course, appTopic, appLevel, topic.type);

        appTopicsAndLevels.levels = [...appTopicsAndLevels.levels, appLevel];
        appTopicsAndLevels.topics = [...appTopicsAndLevels.topics, appTopic];
        appTopicsAndLevels.levelMasteryRecords = [...appTopicsAndLevels.levelMasteryRecords, ...levelMasteryRecords];
      }
    }

    return appTopicsAndLevels;
  }

  private async getMasteryRecordsWriting(course: AppCourse, appTopic: AppTopic, appLevel: AppLevel, type: string) {
    const masteryRecordsUrl =
      type === 'Quick Write'
        ? `https://www.noredink.com/teach/courses/${course.courseId}/quick_writes/${appTopic.topicId}`
        : `https://www.noredink.com/teach/courses/${course.courseId}/guided_drafts/${appTopic.topicId}`;

    const dataElementSelector =
      type === 'Quick Write' ? '#teach-quick-writes-class-elm-flags' : '#teach-courses-guided-drafts-elm-flags';

    const { students, statesById } = (await this._getHTMLElementAttributeASJSON(
      masteryRecordsUrl,
      dataElementSelector,
      'data-flags'
    )) as WritingMasteryLevelsJSON;

    return statesById
      .filter(([, state]) => state === 'submitted' || state === 'graded')
      .map(([id, state]) => {
        const graded = students.find(student => student.id === id || student.studentStateId === id);
        return this.mapper.mapNoredinkWritingStudentGradeToLevelMasteryRecord(
          graded,
          appTopic,
          appLevel,
          state === 'graded'
        );
      });
  }

  private async getTopicsAndLevelsLanguage(courses: AppCourse[]) {
    const pathaways = (await this._getContentOfPageASJSON(
      'https://www.noredink.com/curriculum/api/library'
    )) as NoredinkTopic[];
    const languageCourses: AppCourse[] = [courses.filter(course => course.subjectKeys === 'Language')[0]];
    const appTopicsAndLevels = { topics: [], levels: [], levelMasteryRecords: [] };

    for (let i = 0; i < languageCourses.length; i++) {
      const course = languageCourses[i];
      for (let j = 0; j < pathaways.length; j++) {
        const pathaway = pathaways[j];
        if (!pathaway.topics || pathaway.topics.every(a => a.passage)) {
          continue;
        }

        const appTopic = this.mapper.mapNoredinkTopicToAppTopic(pathaway, course, j + 1);
        const appLevels = pathaway.topics
          .filter(topic => !topic.passage)
          .map((topic, index) => {
            const gradeLevel = pathaway.gradeLevels.sort()[0];
            return this.mapper.mapNoredinkLevelToAppLevel(topic, pathaway.id, index + 1, gradeLevel);
          });
        const levelMasteryRecords = await this.getMasteryRecordsLanguage(course, appTopic);

        appTopicsAndLevels.topics = [...appTopicsAndLevels.topics, appTopic];
        appTopicsAndLevels.levels = [...appTopicsAndLevels.levels, ...appLevels];
        appTopicsAndLevels.levelMasteryRecords = [...appTopicsAndLevels.levelMasteryRecords, ...levelMasteryRecords];
      }
    }
    return appTopicsAndLevels;
  }

  private async getMasteryRecordsLanguage(course: AppCourse, topic: AppTopic) {
    const json = await this._getHTMLElementAttributeASJSON(
      `https://www.noredink.com/teach/courses/${course.courseId}/learning_paths/${topic.topicId}`,
      '#teach-learning-paths-show-flags',
      'data-page'
    );
    let levelMasteryRecords: LevelMasteryRecord[] = [];
    json.students.forEach((student: NoredinkStudent) => {
      const studentRecords = student.topics.map(
        (level: NoredinkStudentTopic): LevelMasteryRecord => ({
          date: new Date().toISOString(),
          credentialId: this.credentials.id,
          levelId: `${level.id}`,
          topicId: `${topic.topicId}`,
          studentId: `${student.id}`,
          activityUnitsAttempted: level.questionsAnswered || 0,
          activityUnitsCorrect: level.questionsAnsweredCorrectly || 0,
          masteryPercentage: 100 * level.completion,
        })
      );
      levelMasteryRecords = [...levelMasteryRecords, ...studentRecords];
    });
    return levelMasteryRecords;
  }
}