Dash Rebuild. M1 Decomposition

m1.all_days_table-definition.ts

const table = new glue.CfnTable(this, 'AllDaysTable', {
      databaseName: database.databaseName,
      catalogId: cdk.Aws.ACCOUNT_ID,
      tableInput: {
        name: 'all_days',
        partitionKeys: [],
        storageDescriptor: {
          columns: [
            { name: 'schoolYear', type: 'string' }
            { name: 'session', type: 'string' },
            { name: 'dayOf', type: 'date' },
            { name: 'weekOf', type: 'string' },
            { name: 'weekNum', type: 'integer' },
            { name: 'isWeekday', type: 'boolean' },
            { name: 'isHoliday', type: 'boolean' },
            { name: 'percentThroughSession', type: 'float' },
            { name: 'isMondayOrAfter', type: 'boolean' },
            { name: 'isFridayOrBefore', type: 'boolean' },
            { name: 'calculatedColumn', type: 'boolean' },
            { name: 'isLincExcluded', type: 'boolean' }
          ],
          location: `s3://alpha-backend-stage-progressrecordstore-1jpwvu35msl4q/all_days`,
          serdeInfo: {
            serializationLibrary: 'org.apache.hadoop.hive.ql.io.orc.OrcSerde',
            parameters: {
              'orc.column.index.access': 'false',
              'orc.compress': 'SNAPPY'
            }
          },
          inputFormat: 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat',
          outputFormat: 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
        },
        parameters: {}
      }
    })

m1.dag.md

flowchart TD
    1["User Migration Lambda.\nUserfront adapters"]
    2["User Migration Lambda.\nCognito adapters"]
    3["User Migration Lambda.\nMain"]
    4["Login Screen"]
    5["Dash API.\nAthena Adapter"]
    6["GET /student\nendpoint"]
    7["GET /time-and-activity\nendpoint"]
    8["GET /antipattern endpoint"]
    9["All Days data sync.\nSheet and S3 adapters"]
    10["All Days data sync.\nMain lambda"]
    11["GET /all-days/day\nendpoint"]
    12["GET /all-days/week\nendpoint"]
    13["API Performance Tests"]
    14["My Daily Activity\nScreen"]
    15["My Weekly Activity\nScreen"]
    16["My Monthly Activity\nScreen"]
    17["M2 Prep"]
    18["Test Results"]
    19["Demo prep"]

    1 --> 3
    2 --> 3
    3 --> 4
    5 --> 6 --> 13
    5 --> 7 --> 13
    5 --> 8 --> 13
    9 --> 10
    10 --> 11 --> 13
    10 --> 12 --> 13
    13 --> 14
    13 --> 15
    13 --> 16
    4 --> 18
    14 --> 18
    15 --> 18
    16 --> 18
    18 --> 19
    13 --> 17 --> 19

Loading

m1.get-antipatterns-definition.ts

export interface GetAntiPatternInput {
	startDate: string
	endDate: string
	students: string[]
}
export type GetAntiPatternOutput = DashApiResponse<AntiPatternItem>
export type DashApiResponse<T> = {
	response: {
		data: {
			json: T[]
		}
	}
}
export interface AntiPatternItem {
	id: string
  student_id: string
  date: string
  student_name: string
  subject: string
  app: string
  summary: string
  antipattern: string
  fix: any
  trigger: any
  links: string
  icon: string
}

m1.get-endpoint-lambda-definition.ts

const athena = new AthenaClient({ region: process.env.AWS_REGION });

export const main = async (proxyEvent: APIGatewayProxyEventV2): Promise<APIGatewayProxyResultV2> {
    const config: AthenaQueryConfig = {
    catalog: 'AwsDataCatalog',
    database: 'alpha_stage',
    table: 'student',
    resultOutput: 's3://athena-query-results-alphamanualcoachbot/Unsaved',
    waitingStrategy: {
      factor: 1.1,
      startMs: 100,
    },
  }; // TODO This is hardcoded example. Call await getConfig() instead.

  const input: StartQueryExecutionCommandInput = {
    QueryString: 'select * from student',
    QueryExecutionContext: {
      Database: config.database,
      Catalog: config.catalog,
    },
    ResultConfiguration: {
      OutputLocation: config.resultOutput,
    },
  };
  const athenaOut: Array<any> = await getResults(athena, config, input);
	const out = mapAthenaOut(athenaOut) // TODO Implement mapAthenaOut to comply with Legacy Dash API Output
	return {
    statusCode: 200,
    body: JSOn.stringify(out)
  }
}

m1.get-session-week-days-definition.ts

export interface GetSessionWeekDaysInput {
	session: number
	week: number
}
export interface SessionWeekDayItem {
	day_of: string
	is_holiday: boolean
	is_weekday: boolean
	percent_through_session: number
	school_year: string
	session: number
	week_num: number
}
export type GetSessionWeekDaysOutput = DashApiResponse<SessionWeekDayItem>
export type DashApiResponse<T> = {
	response: {
		data: {
			json: T[]
		}
	}
}

m1.get-students-definition.ts

export interface GetStudentsInput {
  schoolYear?: string
  campusId?: string
  studentIds?: string[]
  alphaLevel?: number[]
}
export interface StudentItem {
  name: string
  id: string
  student_id: string
  alpha_level: string
}
export type GeStudentsOutput = DashApiResponse<StudentItem>
export type DashApiResponse<T> = {
	response: {
		data: {
			json: T[]
		}
	}
}

m1.get-time-and-activity-definition.ts

export interface GetTimeAndActivityInput {
	startDate: string
	endDate: string
	students: string[]
	schoolYear: string
}
export type GetTimeAndActivityOutput = DashApiResponse<TimeAndActivityItem>
export type DashApiResponse<T> = {
	response: {
		data: {
			json: T[]
		}
	}
}
export interface TimeAndActivityItem {
  date: string
  school_year: string
  session: number
  week_num: number
  student: string
  campus_and_student_id: string
  level: number
  subject: string
  app: string
  course: string
  correct_questions_per_hour: number
  perc_questions_correct: number
  minutes_working: number
  data_source: any
  learning_2x_threshold: number
  correct: number
  total: number
  has_antipattern: boolean
  time_source: string
  should_antipatterns_count_col: boolean
  is_above_mins_threshold: boolean
  is_above_accuracy_threshold: boolean
  is_above_qcph_threshold: boolean
  has_impacting_antipattern: boolean
  count_2x_checks: number
  is_2x_session: boolean
  possible_2x_sessions: number
  possible_2x_checks: number
  id: string
  should_qcph_count: boolean
  combo_multiplier: number
}

m1.get-weeks-definition.ts

export interface GetWeeksInput {
	endDate: string
}
export interface WeekItem {
	week_num: number
	session: number
}
export type GetWeeksOutput = DashApiResponse<WeekItem>
export type DashApiResponse<T> = {
	response: {
		data: {
			json: T[]
		}
	}
}

m1.getting-started.md

Getting Started Guide

Use yarn for everything!

Never do amplify push when you are in staging environment!

Use amplify status to check what is your environment.

After repo OR main branch checkout: ALWAYS

run

yarn install

When you are working on the function: ALWAYS

In the function folder go to lib folder and run yarn install there.
One more time: lib folder.
It's important because if you run yarn install in the function folder, you won't get your dependencies and yarn won't complain.

Example

cd amplify/backend/function/DashAuthPreAuthentication/lib
yarn install

Amplify CLI

AWS Amplify Documentation

Install Amplify CLI and configure it

Follow the Guide here
The alphadashaws app is in 750697792272 (Alpha Dev account)

Create Your Environment from staging environment

See this video

FAQ

What inputs I have to provide to Amplify when I check out the environment?

		? Select the authentication method you want to use: AWS profile
		For more information on AWS Profiles, see:
		https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-profiles.html

		? Please choose the profile you want to use dev-alpha-amplify
		Amplify AppID found: d18zn6pvl31zp7. Amplify App name is: alphadashaws
		Backend environment staging found in Amplify Console app: alphadashaws
		? Choose your default editor: IntelliJ IDEA
		√ Choose the type of app that you're building · javascript
		Please tell us about your project
		? What javascript framework are you using react
		? Source Directory Path:  src
		? Distribution Directory Path: _static
		? Build Command:  yarn build
		? Start Command: yarn start
		? Do you plan on modifying this backend? Yes
		√ Successfully pulled backend environment staging from the cloud.

What do we commit from amplify generated files?

We commit everything that is not ignored

When creating a lambda function it asks if we want to access other resources. Naturally I would answer yes, but I don't see my resource in the Amplify resources.

It means this resource is external for Amplify stack.
Use custom-policies.json to add permissions to such resources.
See last few minutes of this video about how to do that

So our lambdas will be in JavaScript not TS right?

No. They will be Typescript. See this video to learn how to set up it.

How to see the response of the legacy Dash API?

• Go to https://dash.alpha.school/dashboard/daily-activity/
• Login as alpha.coachbot.units@trilogy.com (Iloveschool1!)
• Open Chrome Developer Tools (Network tab)
• Reload https://dash.alpha.school/dashboard/daily-activity/

UPD 06-Jul-23!!! How to set up Typescript Lambda function?

See this video

How do I test and debug my functions locally?

I can share my strategy that uses jest to make real calls to AWS services in Dev account. See this video

m1.trpc-definition.ts

export const trpc = {
	mergedTimeAndActivity: {
		get: {
			useQuery: (input: TimeAndActivityInput) => {
				// TODO implement
			}
		}
	},
	antipatterns: {
		get: {
			useQuery: (input: GetAntipatternInput) => {
				// TODO implement
			}
		}
	},
	allDays: {
		get: {
			useQuery: (input: AllDaysGetInput) => {
				// TODO implement
			}
		},
		getAllWeeks: {
			useQuery: (input: AllWeeksGetInput) => {
				// TODO implement
			}
		}
	},
	masterRoster: {
		getStudents: {
			useQuery: (input: GetStudentsInput) => {

			}
		}
	}
}