Skip to content

Instantly share code, notes, and snippets.

@slickroot
Created December 2, 2022 11:25
Show Gist options
  • Save slickroot/5a718528cc42eb0f0796484f573c1655 to your computer and use it in GitHub Desktop.
Save slickroot/5a718528cc42eb0f0796484f573c1655 to your computer and use it in GitHub Desktop.
import puppeteer, { Browser, Page } from 'puppeteer';
import { LearningAppImportCredentials } from './sdk/learningAppImportStartup';
import { AppCourse, AppLevel, AppTopic, LevelMasteryRecord } from './sdk/learningAppImport';
import * as cheerio from 'cheerio';
interface NoredinkCourse {
id: number;
name: string;
}
interface NoredinkTopicsAndLevelsJSONCourse {
id: number;
grade_indices: number[];
}
interface NoredinkTopicsAndLevelsJSON {
pageState: {
assignments: NoredinkTopic[];
};
courses: NoredinkTopicsAndLevelsJSONCourse[];
}
interface NoredinkWritingStudentGrade {
id: number;
studentStateId: number;
grade: number;
}
interface WritingMasteryLevelsJSON {
students: NoredinkWritingStudentGrade[];
statesById: [id: number, state: string][];
}
interface NoredinkLevel {
id: number;
name: string;
passage: boolean;
}
interface NoredinkTopic {
id: number;
name: string;
type: string;
topics: NoredinkLevel[];
gradeLevels: number[];
}
interface NoredinkStudent {
id: number;
topics: NoredinkStudentTopic[];
}
interface NoredinkStudentTopic {
id: number;
questionsAnswered: number;
questionsAnsweredCorrectly: number;
completion: number;
}
export class NoredinkEntitiesMapper {
credentials: LearningAppImportCredentials;
constructor(credentials: LearningAppImportCredentials) {
this.credentials = credentials;
}
mapNoredinkCourseToAppCourse(course: NoredinkCourse): AppCourse {
return {
credentialId: this.credentials.id,
courseName: course.name,
courseId: `${course.id}`,
subjectKeys: course.name.toLowerCase().includes('writing') ? 'Writing' : 'Language',
};
}
mapNoredinkWritingStudentGradeToLevelMasteryRecord(
student: NoredinkWritingStudentGrade,
appTopic: AppTopic,
appLevel: AppLevel,
graded?: boolean
): LevelMasteryRecord {
return graded
? {
date: new Date().toISOString(),
credentialId: this.credentials.id,
levelId: appLevel.levelId,
topicId: appTopic.topicId,
studentId: `${student.id || student.studentStateId}`,
activityUnitsAttempted: 1,
activityUnitsCorrect: student.grade > 50 ? 1 : 0,
masteryPercentage: student.grade,
}
: {
date: new Date().toISOString(),
credentialId: this.credentials.id,
levelId: appLevel.levelId,
topicId: appTopic.topicId,
studentId: `${student.id || student.studentStateId}`,
activityUnitsAttempted: 1,
activityUnitsCorrect: 0,
masteryPercentage: 0,
};
}
mapNoredinkLevelToAppLevel(
level: NoredinkTopic | NoredinkLevel,
topicId: number,
order: number,
gradeLevel: number
): AppLevel {
return {
credentialId: this.credentials.id,
levelName: level.name,
levelId: `${level.id}`,
gradeLevel: `${gradeLevel}`,
levelOrder: order,
topicId: `${topicId}`,
};
}
mapNoredinkTopicToAppTopic(topic: NoredinkTopic, course: AppCourse, order: number): AppTopic {
return {
credentialId: this.credentials.id,
topicName: topic.name,
topicId: `${topic.id}`,
topicOrder: order,
courseId: `${course.courseId}`,
};
}
}
export class NoredinkAppScrapper {
credentials: LearningAppImportCredentials;
mapper: NoredinkEntitiesMapper;
navigator: Page;
browser: Browser;
constructor(credentials: LearningAppImportCredentials) {
this.credentials = credentials;
this.mapper = new NoredinkEntitiesMapper(credentials);
}
public async init() {
this.browser = await puppeteer.launch({ headless: false, userDataDir: './userdata' });
this.navigator = await this.browser.newPage();
await this.navigator.setViewport({ width: 1920, height: 955 });
await this._login();
}
public async close() {
await this.browser.close();
}
private async _login() {
await this.navigator.goto('https://www.noredink.com/teach/dashboard');
if (this.navigator.url() === 'https://www.noredink.com/teach/dashboard') return;
const loginWithPasswordButtonSelector = '#log_in_with_password';
await this.navigator.click(loginWithPasswordButtonSelector);
const loginFormSelector = '.manual-login-form';
await this.navigator.waitForSelector(loginFormSelector);
const emailInputSelector = '#Nri-Ui-TextInput-Email-or-username';
await this.navigator.type(emailInputSelector, this.credentials.username);
const passwordInputSelector = '#Nri-Ui-TextInput-Password';
await this.navigator.type(passwordInputSelector, this.credentials.password);
const submitButtonSelector = 'button[type=submit]';
await this.navigator.click(`${loginFormSelector} ${submitButtonSelector}`);
await this.navigator.waitForNavigation();
await this.navigator.screenshot({ path: 'screenshot.png' });
}
private async _getHTMLElementAttributeASJSON(url: string, elementId: string, attribute: string) {
const html = await this._getHTMLOfPage(url);
const $ = cheerio.load(html);
return JSON.parse($(elementId).attr(attribute).replaceAll('\n', ''));
}
private async _getContentOfPageASJSON(url: string) {
await this.navigator.goto(url);
const innerText = await this.navigator.evaluate(() => {
return JSON.parse(document.querySelector('body').innerText);
});
return innerText;
}
private async _getHTMLOfPage(url: string) {
await new Promise(resolve => setTimeout(resolve, 1000));
await this.navigator.goto(url);
return this.navigator.content();
}
public async getCourses(): Promise<AppCourse[]> {
const json = await this._getHTMLElementAttributeASJSON(
'https://www.noredink.com/teach/dashboard',
'#teacher-dashboard-elm-flags',
'data-flags'
);
return json.courses.map((course: NoredinkCourse) => this.mapper.mapNoredinkCourseToAppCourse(course));
}
public async getTopicsAndLevels(courses: AppCourse[]) {
const language = await this.getTopicsAndLevelsLanguage(courses);
const writing = await this.getTopicsAndLevelsWriting(courses);
return {
topics: [...language.topics, ...writing.topics],
levels: [...language.levels, ...writing.levels],
levelMasteryRecords: [...language.levelMasteryRecords, ...language.levelMasteryRecords],
};
}
private async getTopicsAndLevelsWriting(courses: AppCourse[]) {
const appTopicsAndLevels = { topics: [], levels: [], levelMasteryRecords: [] };
const noredinkCourses = [courses.filter(course => course.subjectKeys === 'Writing')[0]];
for (let i = 0; i < noredinkCourses.length; i++) {
const course = noredinkCourses[i];
const json = (await this._getHTMLElementAttributeASJSON(
`https://www.noredink.com/teach/courses/${course.courseId}/assignments.html`,
'#assignments-page-elm-flags',
'data-flags'
)) as NoredinkTopicsAndLevelsJSON;
const { assignments } = json.pageState;
for (let j = 0; j < assignments.length; j++) {
const topic = assignments[j];
if (topic.type !== 'Guided Draft' && topic.type !== 'Quick Write') continue;
const appTopic = this.mapper.mapNoredinkTopicToAppTopic(topic, course, j + 1);
const gradeLevel = json.courses.find(c => `${c.id}` === course.courseId).grade_indices.sort()[0];
const appLevel = this.mapper.mapNoredinkLevelToAppLevel(topic, topic.id, j + 1, gradeLevel);
const levelMasteryRecords = await this.getMasteryRecordsWriting(course, appTopic, appLevel, topic.type);
appTopicsAndLevels.levels = [...appTopicsAndLevels.levels, appLevel];
appTopicsAndLevels.topics = [...appTopicsAndLevels.topics, appTopic];
appTopicsAndLevels.levelMasteryRecords = [...appTopicsAndLevels.levelMasteryRecords, ...levelMasteryRecords];
}
}
return appTopicsAndLevels;
}
private async getMasteryRecordsWriting(course: AppCourse, appTopic: AppTopic, appLevel: AppLevel, type: string) {
const masteryRecordsUrl =
type === 'Quick Write'
? `https://www.noredink.com/teach/courses/${course.courseId}/quick_writes/${appTopic.topicId}`
: `https://www.noredink.com/teach/courses/${course.courseId}/guided_drafts/${appTopic.topicId}`;
const dataElementSelector =
type === 'Quick Write' ? '#teach-quick-writes-class-elm-flags' : '#teach-courses-guided-drafts-elm-flags';
const { students, statesById } = (await this._getHTMLElementAttributeASJSON(
masteryRecordsUrl,
dataElementSelector,
'data-flags'
)) as WritingMasteryLevelsJSON;
return statesById
.filter(([, state]) => state === 'submitted' || state === 'graded')
.map(([id, state]) => {
const graded = students.find(student => student.id === id || student.studentStateId === id);
return this.mapper.mapNoredinkWritingStudentGradeToLevelMasteryRecord(
graded,
appTopic,
appLevel,
state === 'graded'
);
});
}
private async getTopicsAndLevelsLanguage(courses: AppCourse[]) {
const pathaways = (await this._getContentOfPageASJSON(
'https://www.noredink.com/curriculum/api/library'
)) as NoredinkTopic[];
const languageCourses: AppCourse[] = [courses.filter(course => course.subjectKeys === 'Language')[0]];
const appTopicsAndLevels = { topics: [], levels: [], levelMasteryRecords: [] };
for (let i = 0; i < languageCourses.length; i++) {
const course = languageCourses[i];
for (let j = 0; j < pathaways.length; j++) {
const pathaway = pathaways[j];
if (!pathaway.topics || pathaway.topics.every(a => a.passage)) {
continue;
}
const appTopic = this.mapper.mapNoredinkTopicToAppTopic(pathaway, course, j + 1);
const appLevels = pathaway.topics
.filter(topic => !topic.passage)
.map((topic, index) => {
const gradeLevel = pathaway.gradeLevels.sort()[0];
return this.mapper.mapNoredinkLevelToAppLevel(topic, pathaway.id, index + 1, gradeLevel);
});
const levelMasteryRecords = await this.getMasteryRecordsLanguage(course, appTopic);
appTopicsAndLevels.topics = [...appTopicsAndLevels.topics, appTopic];
appTopicsAndLevels.levels = [...appTopicsAndLevels.levels, ...appLevels];
appTopicsAndLevels.levelMasteryRecords = [...appTopicsAndLevels.levelMasteryRecords, ...levelMasteryRecords];
}
}
return appTopicsAndLevels;
}
private async getMasteryRecordsLanguage(course: AppCourse, topic: AppTopic) {
const json = await this._getHTMLElementAttributeASJSON(
`https://www.noredink.com/teach/courses/${course.courseId}/learning_paths/${topic.topicId}`,
'#teach-learning-paths-show-flags',
'data-page'
);
let levelMasteryRecords: LevelMasteryRecord[] = [];
json.students.forEach((student: NoredinkStudent) => {
const studentRecords = student.topics.map(
(level: NoredinkStudentTopic): LevelMasteryRecord => ({
date: new Date().toISOString(),
credentialId: this.credentials.id,
levelId: `${level.id}`,
topicId: `${topic.topicId}`,
studentId: `${student.id}`,
activityUnitsAttempted: level.questionsAnswered || 0,
activityUnitsCorrect: level.questionsAnsweredCorrectly || 0,
masteryPercentage: 100 * level.completion,
})
);
levelMasteryRecords = [...levelMasteryRecords, ...studentRecords];
});
return levelMasteryRecords;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment