Skip to content

Instantly share code, notes, and snippets.

@parvaurea
Last active October 6, 2020 00:01
Show Gist options
  • Select an option

  • Save parvaurea/23e9403c353c2c22ef4c2151d9f5362f to your computer and use it in GitHub Desktop.

Select an option

Save parvaurea/23e9403c353c2c22ef4c2151d9f5362f to your computer and use it in GitHub Desktop.

5k-pattern-quicksight-embedding

  • Report Creation - Manual Step

  • Report Deployment Scripts

    • Making artifacts created in step-1 part of code repository

      • Sharing artifacts with cli user

        Normally at the structure is in AWS Rewrite projects. The user accounts are connected to adfs. So every dev creates a cli user. If the reports are not shared with the cli user then. You won’t be able to execute commands related to description of the artifacts
    • directory structure

      quicksight
        templates
          analysis-templates
            report-analysis-1.json
          dashboards
            report-dashboard-1.json
          data-sets
            data-set-1.json
      
    • Deployment scripts

      • creating data source

        • copying the data source created
          aws quicksight describe-data-source --data-source-id <data-source-id>--aws-account-id <aws-account-id>
        • modifying and creating mustache template
          • copy the response.DataSource to a separate json file
          • make the json to match this structure
            await this.awsQuicksight.createDataSource({
            // AWS_ACCOUNT_ID
            AwsAccountId: awsAccountId,
            // Created dtaa source id Ex. <env>_<data_source_name>
            DataSourceId: dataSourceId,
            Name: `${stackPrefix} ${context.envType} Reporting DataSource RDS`,
            Type: 'AURORA',
            Credentials: {
              CredentialPair: {
                 // got from stack outputs
                Username: rdsUsername,
                // got from stack outputs
                Password: rdsPassword, 
              },
            },
            DataSourceParameters: {
              AuroraParameters: {
                // got from stackout put
                Database: dbName,
                // got from stack outputs
                Host: stackOutputs[PrefixHandler.handle(context.envType, 'RdsClusterHost')] as string,
                // got from stack outputs
                Port: Number(stackOutputs[PrefixHandler.handle(context.envType, 'RdsClusterPort')]),
              },
            },
            // superUserArns - just a static list of users this entity should be shared with
            Permissions: superUserArns.map(arn => ({
              Principal: arn,
              Actions: [
                'quicksight:UpdateDataSourcePermissions',
                'quicksight:DescribeDataSource',
                'quicksight:DescribeDataSourcePermissions',
                'quicksight:PassDataSource',
                'quicksight:UpdateDataSource',
                'quicksight:DeleteDataSource',
              ],
            })),
            }).promise();
    • Organization / customer creation scripts

      Every Template being used below wil be created for every organization To ensure only the current organization / cutomer can access its own data filtering in done on datasets

      • creating data set

        • copying the data set created
          aws quicksight describe-data-set --data-set-id <data-set-id> --aws-account-id <aws-account-id>
        • modifying and creating mustache template
          • copy the response.DataSource to a separate json file
          • make the json to match this structure
            {
              "DataSetId": "{{{DataSetId}}}", // created data source id Ex. orgId_<data_set_name>
              "Name": "{{{DataSetName}}}", // <org_id> <data_set_name>
              "PhysicalTableMap": {
                "5cd7ef53-64db-4c67-87b8-ef4667a7f437": {
                  "RelationalTable": {
                    "DataSourceArn": "{{{RedshiftDataSourceArn}}}", // got from stack output
                    "Schema": "public",
                    "Name": "{{{RedshiftReportingViewName}}}", // hardcoded from migrations
                    "InputColumns": // same as in cli response
                  }
                },
                "75fdea21-aca5-49d5-99d8-23cf72ab2adf": {
                  "RelationalTable": {
                    "DataSourceArn": "{{{AuroraDataSourceArn}}}", // got from stack output
                    "Name": "{{{RDSReportingViewName}}}", // hardcoded from migrations
                    "InputColumns": // same as in cli response
                  }
                }
              },
              "LogicalTableMap": {
                "75fdea21-aca5-49d5-99d8-23cf72ab2adf": {
                  "Alias": "{{{RDSReportingViewName}}}", // hardcoded from migrations
                  "Source": {
                    "PhysicalTableId": "75fdea21-aca5-49d5-99d8-23cf72ab2adf"
                  }
                },
                "88385f85-1b68-4553-8ae5-23b784adb455": {
                  "Alias": "Intermediate Table",
                  // same as in the cli response - except for shown properties
                  "DataTransforms": [
                    {
                      "FilterOperation": {
                        // org_id - got uplon adding to the db or created
                        "ConditionExpression": "{org_id}=\"{{{org_id}}}\""
                      }
                    },
                  ],
                  "Source": // same as in response
                "927c8831-1bd7-4ad9-b170-000341987617": {
                  "Alias": "{{{RedshiftReportingViewName}}}", // hardcoded from migrations
                  "DataTransforms": , // same as in reponse
                  "Source": {
                    "PhysicalTableId": "5cd7ef53-64db-4c67-87b8-ef4667a7f437"
                  }
                }
              },
              "ImportMode": "SPICE"
            }
          • save in quicksight > templates > data-sets > data-set-1.json
      • creating analysis template

        • modifying and creating mustache template
          • copy paste the following json structure and replace the values
            {
              "AwsAccountId": "{{{AwsAccountId}}}",
              "TemplateId": "{{{TemplateId}}}",
              "Name": "{{{TemplateName}}}",
              "SourceEntity": {
                "SourceAnalysis": {
                  // this value will be hardcoded has this is. This is because it points to the repors
                  // created in step 1
                  "Arn": "arn:aws:quicksight:<AWS_REGION>:<AWS_ACCOUNT_ID>:analysis/<ANALYSIS_ID>",
                  // All the datasets being using will be listed here
                  "DataSetReferences": [
                    {
                      "DataSetPlaceholder": "DS1",
                      // this value will be hardcoded has this is. This is because it points to the repors
                      // created in step 1
                      "DataSetArn": "arn:aws:quicksight:<AWS_REGION>:<AWS_ACCOUNT_ID>:dataset/<DATA_SET_ID>"
                    }
                  ]
                }
              },
              "VersionDescription": "1"
          }
          • save in quicksight > templates > analysis-templates > report-analysis-1.json
      • creating dashbard

        • copying the data source created
        • modifying and creating mustache template
          • copy the response.DataSource to a separate json file
          aws quicksight describe-dashboard --dashboard-id <dashboard-id> --aws-account-id <aws-account-id>
          • make the json to match this structure
          {
            "AwsAccountId": "{{{AwsAccountId}}}",
            "DashboardId": "{{{DashboardId}}}",
            // created through script - Ex. <org_id>-report-dashboard-1.json
            "Name": "{{{DashboardName}}}",
            "SourceEntity": {
              "SourceTemplate": {
                "DataSetReferences": [
                  {
                    "DataSetPlaceholder": "DS1",
                    // name of the data-set file created
                    "DataSetArn": "{{{data-set-1}}}"
                  }
                ],
                // name of the analysis file created
                "Arn": "{{{report-analysis-1}}}"
              }
            },
            "VersionDescription": "1",
            "DashboardPublishOptions": {
              "AdHocFilteringOption": {
                "AvailabilityStatus": "DISABLED"
              },
              "ExportToCSVOption": {
                "AvailabilityStatus": "ENABLED"
              },
              "SheetControlsOption": {
                "VisibilityState": "EXPANDED"
              }
            }
          }
          • save in quicksight > templates > dashboards > report-dashboard-1.json
      • deployment scripts

      The related scripts and cliQuicksight scripts is present at https://gist.github.com/parvaurea/23e9403c353c2c22ef4c2151d9f5362f

      The basic flow of deployment scripts is as follows - this is a subset of create customer / organization

      console.log('[*] creating quicksight group');
      const group = await this.cliQuicksight.createGroup(orgId);
      console.log('[*] created quicksight group');
      
      console.log('[*] getting super user arns');
      const superUserArns = await this.cliQuicksight.getUserArns(this.config.organization.superUsers);
      console.log('[*] got super user arns');
      
      console.log('[*] creating quicksight reporting data sets');
      await this.cliQuicksight.createDatasets(
        env,
        envType,
        orgId,
        pinpointApp.Id,
        group,
        superUserArns,
        tagsArray,
      );
      console.log('[*] created quicksight reporting data sets');
      
      console.log('[*] creating quicksight reporting templates');
      await this.cliQuicksight.createTemplates(orgId, group, superUserArns);
      console.log('[*] created quicksight reporting templates');
      
      console.log('[*] creating quicksight dashboards');
      const dashboards = await this.cliQuicksight.createDashboards(pinpointApp.Id, group, superUserArns, orgId);
      console.log('[*] created quicksight dashboards');
      
      console.log('[*] creating quicksight dashboards access role');
      const generateReportLambdaRoleArn = await this.cliLambda.getLambdaRoleArn('generateReportUrl');
      const quicksightAccessRole = await this.cliQuicksight.createDashboardAccessRole(
        pinpointApp.Id,
        dashboards,
        generateReportLambdaRoleArn,
        tagsArray,
      );
      console.log(`[*] created quicksight dashboards access role : ${quicksightAccessRole.Arn}`);
      
      console.log('[*] creating quicksight user');
      const quicksightUser = await this.cliQuicksight.createUser(user.email, quicksightAccessRole.Arn);
      console.log('[*] created quicksight user');
      
      const username = quicksightUser?.UserName || `${quicksightAccessRole.RoleName}/${user.email}`;
      console.log('[*] adding quicksight user to group');
      await this.cliQuicksight.addUserToGroup(username, group.GroupName as string);
      console.log('[*] added quicksight user to group');
  • Report Frontend Embedding

    • Angular service
    import { DOCUMENT } from '@angular/common';
    import { Inject, Injectable } from '@angular/core';
    import * as QuicksightEmbedding from 'amazon-quicksight-embedding-sdk';
    import { Apollo } from 'apollo-angular';
    import gql from 'graphql-tag';
    import { Observable } from 'rxjs';
    import { map } from 'rxjs/operators';
    
    @Injectable({
      providedIn: 'root',
    })
    export class ReportsService {
      public readonly allParamValue = '[ALL]';
    
      constructor(
        private readonly apollo: Apollo,
        @Inject(DOCUMENT) private readonly document: Document,
        @Inject(Window) private readonly window: Window,
      ) { }
    
      public generateReportUrl(dashboardId: string): Observable<string> {
        return this.apollo
          .mutate<{ generateReportUrl: string }>({
            mutation: gql`
              mutation generateReportUrl($dashboardId: String!) {
                generateReportUrl(DashboardId: $dashboardId)
              }
            `,
            variables: {
              dashboardId,
            },
          })
          .pipe(map(res => res.data.generateReportUrl));
      }
    
      public renderReport(dashboardId: string, elementContainerId: string, params?: any): Observable<void> {
        return this.generateReportUrl(dashboardId).pipe(
          map(url => {
            const container = this.document.getElementById(elementContainerId);
            container.innerHTML = '';
            const dashboard = QuicksightEmbedding.embedDashboard({
              url,
              container,
              ...this.reportsConfig.default,
              ...params || {},
              parameters: {
                baseUrl: `${this.window.location.hostname}:${this.window.location.port}`,
                ...params?.parameters || {},
              }
            });
            dashboard.on('error', () => console.log('error rendering report'));
          }),
        );
      }
    }
import { AWSError, IAM, QuickSight } from 'aws-sdk';
import { CreateDashboardResponse, Dashboard } from 'aws-sdk/clients/quicksight';
import * as fs from 'fs-extra';
import * as mustache from 'mustache';
import * as path from 'path';
import { AwsTag } from '../aws-tag.model';
import { CliConfig } from '../cli-config.model';
import { PrefixHandler } from '../../../../lib/models/prefix-handler.model';
import { CliShared } from '../shared.models';
export class CliQuicksightDashboards {
private readonly awsIam = new IAM();
private readonly awsQuicksight = new QuickSight();
private readonly templatesDir = path.join(__dirname, '../../quicksight/templates');
constructor(private readonly config: CliConfig, private readonly cliShared: CliShared) { }
public async createDashboardAccessRole(
pinpointAppId: string,
dashboards: Dashboard[],
generateReportLambdaRoleArn: string,
tags: AwsTag[],
): Promise<IAM.Role> {
const roleName = PrefixHandler.handle(pinpointAppId, 'dashboards-access-role');
let role: IAM.Role;
try {
const res = await this.awsIam
.createRole({
RoleName: roleName,
AssumeRolePolicyDocument: JSON.stringify({
Version: '2012-10-17',
Statement: {
Effect: 'Allow',
Principal: {
AWS: [await this.cliShared.getCurrentUserArn(), generateReportLambdaRoleArn],
},
Action: ['sts:AssumeRole'],
},
}),
Tags: tags,
})
.promise();
role = res.Role;
} catch (err) {
if ((err as AWSError).code !== 'EntityAlreadyExists') {
throw err;
}
const res = await this.awsIam
.getRole({
RoleName: roleName,
})
.promise();
role = res.Role;
}
console.log(`[*] role create : ${role.Arn}`);
console.log(`[*] attaching role permissions`);
await this.awsIam
.putRolePolicy({
RoleName: roleName,
PolicyName: 'dashboard-access',
PolicyDocument: JSON.stringify({
Version: '2012-10-17',
Statement: [
{
Effect: 'Allow',
Action: ['quicksight:RegisterUser'],
Resource: '*',
},
{
Effect: 'Allow',
Action: ['quicksight:GetDashboardEmbedUrl'],
Resource: dashboards.map(dashboard => dashboard.Arn),
},
],
}),
})
.promise();
console.log(`[*] attached role permissions`);
return role;
}
public async createDashboards(
orgId: string,
pinpointAppId: string,
group: QuickSight.Group,
superUserArns: string[],
): Promise<Dashboard[]> {
const localOrgId = orgId || this.config.organization.orgId;
const awsAccountId = await this.cliShared.getAccountId();
const files = fs
.readdirSync(path.join(this.templatesDir, 'dashboards'))
.map(file => path.parse(file).name);
const promises: Promise<Dashboard>[] = [];
const existingDashboards = await this.listDashboards(pinpointAppId, awsAccountId);
for (const file of files) {
const dashboardId = this.getDashboardId(pinpointAppId, file);
const existingDashboard = existingDashboards
.find(dashboard => dashboard.DashboardId === dashboardId);
if (!existingDashboard) {
try {
await this.createDashboard(
path.basename(file),
awsAccountId,
group,
superUserArns,
localOrgId,
pinpointAppId,
);
continue;
} catch (error) {
if ((error as AWSError).code !== 'ResourceExistsException') {
throw error;
}
console.log(`[*] dashboard exists ${dashboardId}`);
}
} else {
console.log(`[*] dashboard exists ${dashboardId}`);
}
promises.push(this.getDashboard(dashboardId, awsAccountId));
}
return Promise.all(promises);
}
public async listDashboards(pinpointAppId: string, awsAccountId: string) {
let next: string | undefined;
const ret = [];
do {
const res = await this.awsQuicksight.listDashboards({
AwsAccountId: awsAccountId,
NextToken: next
}).promise();
next = res.NextToken;
const dashboards = (res.DashboardSummaryList || [])
.filter(dashboard => dashboard.DashboardId?.toLowerCase().startsWith(pinpointAppId.toLowerCase()));
ret.push(...dashboards);
} while (next);
return ret;
}
private async createDashboard(
file: string,
awsAccountId: string,
group: QuickSight.Group,
superUserArns: string[],
orgId: string,
pinpointAppId: string,
): Promise<CreateDashboardResponse> {
const localOrgId = orgId || this.config.organization.orgId;
const dashboardId = this.getDashboardId(pinpointAppId, file);
console.log(`[*] creating dashboard ${dashboardId}`);
const template = fs.readFileSync(path.join(this.templatesDir, 'dashboards', `${file}.json`)).toString();
const dashboardName = `${localOrgId} ${file}`;
const view: { [key: string]: string } = {
AwsAccountId: awsAccountId,
DashboardId: dashboardId,
DashboardName: dashboardName,
};
const dataSetTemplateFiles = fs
.readdirSync(path.join(this.templatesDir, 'data-sets'))
.map(file => path.parse(file).name);
for (const templateFile of dataSetTemplateFiles) {
const dataSetId = this.cliShared.getDatasetId(localOrgId, templateFile);
const dataSet = await this.awsQuicksight
.describeDataSet({
AwsAccountId: awsAccountId,
DataSetId: dataSetId,
})
.promise();
view[templateFile] = dataSet.DataSet?.Arn as string;
}
const templateTemplateFiles = fs
.readdirSync(path.join(this.templatesDir, 'analysis-templates'))
.map(file => path.parse(file).name);
for (const templateFile of templateTemplateFiles) {
const templateId = this.cliShared.getTemplateId(localOrgId, templateFile);
const templateDes = await this.awsQuicksight
.describeTemplate({
AwsAccountId: awsAccountId,
TemplateId: templateId,
})
.promise();
view[templateFile] = templateDes.Template?.Arn as string;
}
const templateJson = JSON.parse(mustache.render(template, view));
const dashboard = await this.awsQuicksight
.createDashboard({
...templateJson,
AwsAccountId: awsAccountId,
DashboardId: dashboardId,
Name: dashboardName,
Permissions: [...superUserArns, group.Arn].map(arn => ({
Principal: arn,
Actions: [
'quicksight:DescribeDashboard',
'quicksight:ListDashboardVersions',
'quicksight:UpdateDashboardPermissions',
'quicksight:QueryDashboard',
'quicksight:UpdateDashboard',
'quicksight:DeleteDashboard',
'quicksight:DescribeDashboardPermissions',
'quicksight:UpdateDashboardPublishedVersion',
],
})),
})
.promise();
console.log(`[*] created dashboard ${dashboardId}`);
return dashboard;
}
private async getDashboard(id: string, awsAccountId: string): Promise<Dashboard> {
const res = await this.awsQuicksight
.describeDashboard({
AwsAccountId: awsAccountId,
DashboardId: id,
})
.promise();
return res.Dashboard as Dashboard;
}
private getDashboardId(pinpointAppId: string, file: string) {
return `${pinpointAppId}_${file}`;
}
}
import { AWSError, QuickSight } from 'aws-sdk';
import * as fs from 'fs-extra';
import * as mustache from 'mustache';
import * as path from 'path';
import { AwsTag } from '../aws-tag.model';
import { CliShared } from '../shared.models';
import { CliQuicksightDataSources } from './dataSources.quicksight.models';
export class CliQuicksightDataSets {
private awsQuicksight = new QuickSight();
private templatesDir = path.join(__dirname, '../../quicksight/templates');
constructor(
private readonly cliShared: CliShared,
private readonly cliDataSources: CliQuicksightDataSources,
) { }
public async createDatasets(
env: string,
envType: string,
orgId: string,
pinpointAppId: string,
group: QuickSight.Group,
superUserArns: string[],
tags: AwsTag[],
): Promise<void> {
const awsAccountId = await this.cliShared.getAccountId();
const auroraDataSource = await this.cliDataSources.getQuickSightDataSource(
this.cliDataSources.getQuicksightAuroraDataSourceId(env, envType),
);
const redshiftDataSource = await this.cliDataSources.getQuickSightDataSource(
this.cliDataSources.getQuicksightRedshiftDataSourceId(env, envType),
);
const files = fs
.readdirSync(path.join(this.templatesDir, 'data-sets'))
.map(file => path.parse(file).name);
const promises = [];
for (const file of files) {
promises.push(
this.createQuicksightDataset(
env,
envType,
path.basename(file),
orgId,
pinpointAppId,
awsAccountId,
auroraDataSource?.Arn as string,
redshiftDataSource?.Arn as string,
group,
superUserArns,
tags,
),
);
}
await Promise.all(promises);
}
public async removeQuicksightDataset(orgId: string): Promise<void> {
const awsAccountId = await this.cliShared.getAccountId();
const dataSetId = `${orgId}_email_events_set`;
await this.awsQuicksight
.deleteDataSet({
AwsAccountId: awsAccountId,
DataSetId: dataSetId,
})
.promise();
}
private async createQuicksightDataset(
env: string,
envType: string,
file: string,
orgId: string,
pinpointAppId: string,
awsAccountId: string,
auroraDatasourceArn: string,
redshiftDataSourceArn: string,
group: QuickSight.Group,
superUserArns: string[],
tags: AwsTag[],
) {
const template = fs.readFileSync(path.join(this.templatesDir, 'data-sets', `${file}.json`)).toString();
const dataSetId = this.cliShared.getDatasetId(orgId, file);
console.log(`[*] creating data set ${dataSetId}`);
const dataSetName = `${orgId} ${file}`;
const templateJson = JSON.parse(
mustache.render(template, {
DataSetId: dataSetId,
AwsAccountId: awsAccountId,
DataSetName: dataSetName,
AuroraDataSourceArn: auroraDatasourceArn,
RedshiftDataSourceArn: redshiftDataSourceArn,
PinpointAppId: pinpointAppId,
RedshiftReportingViewName: this.cliShared.getRedshiftReportingViewName(env, envType),
RDSReportingViewName: 'reporting_view_rds',
RedshiftContactsReportingViewName: this.cliShared.getRedshiftContactsReportingViewName(env, envType),
RDSMailingListsTableName: this.cliShared.getRDSMailingListsTableName(),
RedshiftPublicSchemaName: 'public',
RedshiftSuppressionEmailsReportingViewName: this.cliShared.getRedshiftSuppressionEmailsReportingViewName(
env,
envType,
),
RDSSuppressionListsTableName: 'suppression_lists',
RDSUserLoginsTableName: 'user_logins',
RDSLandingPagesTableName: 'landing_pages',
RDSAppsTableName: 'apps'
}),
);
try {
await this.awsQuicksight
.createDataSet({
AwsAccountId: awsAccountId,
DataSetId: dataSetId,
Name: dataSetName,
...templateJson,
Tags: tags,
Permissions: [...superUserArns, group.Arn].map(arn => ({
Principal: arn,
Actions: [
'quicksight:UpdateDataSetPermissions',
'quicksight:DescribeDataSet',
'quicksight:DescribeDataSetPermissions',
'quicksight:PassDataSet',
'quicksight:DescribeIngestion',
'quicksight:ListIngestions',
'quicksight:UpdateDataSet',
'quicksight:DeleteDataSet',
'quicksight:CreateIngestion',
'quicksight:CancelIngestion',
],
})),
})
.promise();
} catch (error) {
const awsError = error as AWSError;
if (awsError?.code === 'ResourceExistsException') {
console.log(`[*] ${awsError.message}`);
} else {
throw error;
}
}
console.log(`[*] created data set ${dataSetId}`);
}
}
import { QuickSight } from 'aws-sdk';
import * as context from '../../../../cdk.context.json';
import { PrefixHandler } from '../../../../lib/models/prefix-handler.model';
import { CliConfig } from '../cli-config.model';
import { CliShared } from '../shared.models';
import { CliStacks } from '../stacks.model';
export class CliQuicksightDataSources {
private readonly awsQuicksight = new QuickSight();
constructor(
private readonly config: CliConfig,
private readonly cliShared: CliShared,
private readonly cliStacks: CliStacks,
) {}
public async createDataSources(stackPrefix: string, superUserArns: string[]): Promise<void> {
const localStackPrefix = stackPrefix || this.config.stackPrefix;
console.log('[*] creating data source Aurora');
await this.addQuicksightAuroraDataSource(localStackPrefix, superUserArns);
console.log('[*] created data source Aurora');
console.log('[*] creating data source Redshift');
await this.addQuicksightRedshiftDataSource(localStackPrefix, superUserArns);
console.log('[*] created data source Redshift');
}
public async getQuickSightDataSource(dataSourceId: string): Promise<QuickSight.DataSource | undefined> {
const existingItem = (
await this.awsQuicksight
.listDataSources({
AwsAccountId: await this.cliShared.getAccountId(),
})
.promise()
).DataSources?.find(ds => ds.DataSourceId === dataSourceId);
return existingItem;
}
public getQuicksightRedshiftDataSourceId(stackPrefix: string, envType: string) {
return `${stackPrefix}_${envType}_redshift`;
}
public getQuicksightAuroraDataSourceId(stackPrefix: string, envType: string): string {
return `${stackPrefix}_${envType}_rds`;
}
private async addQuicksightAuroraDataSource(stackPrefix: string, superUserArns: string[]): Promise<void> {
const awsAccountId = await this.cliShared.getAccountId();
const stackOutputs = await this.cliStacks.getStackOutputs(
PrefixHandler.handle(context.envType, `${context.stackName}RDS`),
);
const { rdsUsername, rdsPassword, rdsDatabaseName } = await this.cliShared.getSecrets();
const dbName = this.cliShared.getRdsDatabaseName(stackPrefix, rdsDatabaseName);
const dataSourceId = this.getQuicksightAuroraDataSourceId(stackPrefix, context.envType);
const dataSource = await this.getQuickSightDataSource(dataSourceId);
if (!dataSource) {
console.log(`[*] creating data source : ${dataSourceId}`);
await this.awsQuicksight
.createDataSource({
AwsAccountId: awsAccountId,
DataSourceId: dataSourceId,
Name: `${stackPrefix} ${context.envType} Reporting DataSource RDS`,
Type: 'AURORA',
Credentials: {
CredentialPair: {
Username: rdsUsername,
Password: rdsPassword,
},
},
DataSourceParameters: {
AuroraParameters: {
Database: dbName,
Host: stackOutputs[PrefixHandler.handle(context.envType, 'RdsClusterHost')] as string,
Port: Number(stackOutputs[PrefixHandler.handle(context.envType, 'RdsClusterPort')]),
},
},
Permissions: superUserArns.map(arn => ({
Principal: arn,
Actions: [
'quicksight:UpdateDataSourcePermissions',
'quicksight:DescribeDataSource',
'quicksight:DescribeDataSourcePermissions',
'quicksight:PassDataSource',
'quicksight:UpdateDataSource',
'quicksight:DeleteDataSource',
],
})),
})
.promise();
} else {
console.log(`[*] data source already exists : ${dataSource.DataSourceId}`);
}
await this.waitForQuicksightDataSourceCreation(dataSourceId);
}
private async addQuicksightRedshiftDataSource(stackPrefix: string, superUserArns: string[]) {
const awsAccountId = await this.cliShared.getAccountId();
const stackOutputs = await this.cliStacks.getStackOutputs(
PrefixHandler.handle(context.envType, `${context.stackName}RedshiftCluster`),
);
const { redshiftMasterUsername, redshiftMasterPassword } = await this.cliShared.getSecrets();
const clusterName = PrefixHandler.handle(context.envType, 'redshift-cluster');
const dbName = `${clusterName}-db`;
const dataSourceId = this.getQuicksightRedshiftDataSourceId(stackPrefix, context.envType);
const dataSource = await this.getQuickSightDataSource(dataSourceId);
if (!dataSource) {
console.log(`[*] creating data source : ${dataSourceId}`);
await this.awsQuicksight
.createDataSource({
AwsAccountId: awsAccountId,
DataSourceId: dataSourceId,
Name: `${stackPrefix} ${context.envType} Reporting DataSource Data Lake`,
Type: 'REDSHIFT',
Credentials: {
CredentialPair: {
Username: redshiftMasterUsername,
Password: redshiftMasterPassword,
},
},
DataSourceParameters: {
RedshiftParameters: {
Database: dbName,
ClusterId: clusterName,
Host: stackOutputs[PrefixHandler.handle(context.envType, 'RedshiftClusterHost')],
Port: Number(stackOutputs[PrefixHandler.handle(context.envType, 'RedshiftClusterPort')]),
},
},
Permissions: superUserArns.map(arn => ({
Principal: arn,
Actions: [
'quicksight:UpdateDataSourcePermissions',
'quicksight:DescribeDataSource',
'quicksight:DescribeDataSourcePermissions',
'quicksight:PassDataSource',
'quicksight:UpdateDataSource',
'quicksight:DeleteDataSource',
],
})),
})
.promise();
} else {
console.log('[*] data source already exists');
}
await this.waitForQuicksightDataSourceCreation(dataSourceId);
}
// returns a variable or type promise
// tslint:disable-next-line: promise-function-async
private waitForQuicksightDataSourceCreation(
dataSourceId: string,
): Promise<QuickSight.DataSource | undefined> {
let dsStatus: string | undefined;
return new Promise<QuickSight.DataSource | undefined>(async res => {
let dataSource: QuickSight.DataSource | undefined;
do {
dataSource = (
await this.awsQuicksight
.describeDataSource({
AwsAccountId: await this.cliShared.getAccountId(),
DataSourceId: dataSourceId,
})
.promise()
).DataSource;
dsStatus = dataSource?.Status;
console.log(`[*] status : ${dsStatus}, waiting to complete`);
await new Promise(resolve => setTimeout(resolve, 1000));
} while (dsStatus === 'CREATION_IN_PROGRESS');
console.log(`[*] data source created`);
res.call(this, dataSource);
});
}
}
import { IAM, QuickSight } from 'aws-sdk';
import { Dashboard } from 'aws-sdk/clients/quicksight';
import { AwsTag } from './aws-tag.model';
import { CliConfig } from './cli-config.model';
import { CliQuicksightDashboards } from './quicksight/dashboards.quicksight.models';
import { CliQuicksightDataSets } from './quicksight/datasets.quicksight.models';
import { CliQuicksightDataSources } from './quicksight/dataSources.quicksight.models';
import { CliQuicksightTemplates } from './quicksight/templates.quicksight.models';
import { CliQuicksightUsers } from './quicksight/users.quicksight.models';
import { CliShared } from './shared.models';
import { CliStacks } from './stacks.model';
export class CliQuicksight {
private cliDashboards: CliQuicksightDashboards;
private cliDataSources: CliQuicksightDataSources;
private cliDataSets: CliQuicksightDataSets;
private cliUsers: CliQuicksightUsers;
private cliTemplates: CliQuicksightTemplates;
constructor(private config: CliConfig, cliShared: CliShared, cliStacks: CliStacks) {
this.cliDashboards = new CliQuicksightDashboards(config, cliShared);
this.cliDataSources = new CliQuicksightDataSources(config, cliShared, cliStacks);
this.cliDataSets = new CliQuicksightDataSets(cliShared, this.cliDataSources);
this.cliUsers = new CliQuicksightUsers(config, cliShared);
this.cliTemplates = new CliQuicksightTemplates(cliShared);
}
public getUserArns = async (users: string[]) => this.cliUsers.getUserArns(users);
public createDashboardAccessRole = (
pinpointAppId: string,
dashboards: Dashboard[],
generateReportLambdaRoleArn: string,
tags: AwsTag[],
): Promise<IAM.Role> =>
this.cliDashboards.createDashboardAccessRole(
pinpointAppId,
dashboards,
generateReportLambdaRoleArn,
tags,
);
public createDashboards = (
pinpointAppId: string,
group: QuickSight.Group,
superUserArns: string[],
orgId: string = this.config.organization.orgId,
): Promise<Dashboard[]> => this.cliDashboards.createDashboards(orgId, pinpointAppId, group, superUserArns);
public createDataSources = async (
stackPrefix = this.config.stackPrefix,
superUsers: string[] = this.config.organization.superUsers,
): Promise<void> => {
const arns = await this.getUserArns(superUsers);
this.cliDataSources.createDataSources(stackPrefix, arns);
};
public createDatasets = (
env: string,
envType: string,
orgId: string,
pinpointAppId: string,
group: QuickSight.Group,
superUserArns: string[],
tags: AwsTag[],
): Promise<void> =>
this.cliDataSets.createDatasets(env, envType, orgId, pinpointAppId, group, superUserArns, tags);
public createUser = (email: string, dashboardAccessRoleArn: string): Promise<QuickSight.User | null> =>
this.cliUsers.createUser(email, dashboardAccessRoleArn);
public addUserToGroup = (username: string, groupName: string): Promise<void> =>
this.cliUsers.addUserToGroup(username, groupName);
public createGroup = (orgId: string = this.config.organization.orgId): Promise<QuickSight.Group> =>
this.cliUsers.createGroup(orgId);
public createTemplates = (orgId: string, group: QuickSight.Group, superUserArns: string[]) =>
this.cliTemplates.createTemplates(orgId, group, superUserArns);
}
import { DOCUMENT } from '@angular/common';
import { Inject, Injectable } from '@angular/core';
import * as QuicksightEmbedding from 'amazon-quicksight-embedding-sdk';
import { Apollo } from 'apollo-angular';
import gql from 'graphql-tag';
import { Observable } from 'rxjs';
import { map } from 'rxjs/operators';
@Injectable({
providedIn: 'root',
})
export class ReportsService {
public readonly reportsConfig = {
default: {
loadingHeight: '700px',
scroll: 'no',
scrolling: 'no',
height: 'AutoFit',
width: '100%',
footerPaddingEnabled: false,
},
suppressionListChart: {
height: '350px',
loadingHeight: '350px',
},
listChart: {
height: '350px',
loadingHeight: '350px',
},
mostRecentlySentMessages: {
loadingHeight: '570px',
height: '570px',
scroll: 'no',
},
myStats: {
loadingHeight: '570px',
height: '570px',
},
messageOverviewDetails: {
height: '300px',
loadingHeight: '300px',
},
standardMessage: {
loadingHeight: '700px',
height: '700px',
},
visualMessage: {
loadingHeight: '700px',
height: '700px',
},
aggregateMessageReport: {
loadingHeight: '400px',
height: '400px',
},
globalDeliveryReport: {
loadingHeight: '700px',
},
overview: {
loadingHeight: '700px',
height: '700px',
}
};
public readonly allParamValue = '[ALL]';
constructor(
private readonly apollo: Apollo,
@Inject(DOCUMENT) private readonly document: Document,
@Inject(Window) private readonly window: Window,
) { }
public generateReportUrl(dashboardId: string): Observable<string> {
return this.apollo
.mutate<{ generateReportUrl: string }>({
mutation: gql`
mutation generateReportUrl($dashboardId: String!) {
generateReportUrl(DashboardId: $dashboardId)
}
`,
variables: {
dashboardId,
},
})
.pipe(map(res => res.data.generateReportUrl));
}
public renderReport(dashboardId: string, elementContainerId: string, params?: any): Observable<void> {
return this.generateReportUrl(dashboardId).pipe(
map(url => {
const container = this.document.getElementById(elementContainerId);
container.innerHTML = '';
const dashboard = QuicksightEmbedding.embedDashboard({
url,
container,
...this.reportsConfig.default,
...params || {},
parameters: {
baseUrl: `${this.window.location.hostname}:${this.window.location.port}`,
...params?.parameters || {},
}
});
dashboard.on('error', () => console.log('error rendering report'));
}),
);
}
}
import { AWSError, QuickSight } from 'aws-sdk';
import * as fs from 'fs-extra';
import * as mustache from 'mustache';
import * as path from 'path';
import { CliShared } from '../shared.models';
export class CliQuicksightTemplates {
private templatesDir = path.join(__dirname, '../../quicksight/templates');
private awsQuicksight = new QuickSight();
constructor(private readonly cliShared: CliShared) { }
public async createTemplates(orgId: string, group: QuickSight.Group, superUserArns: string[]) {
const awsAccountId = await this.cliShared.getAccountId();
const files = fs.readdirSync(path.join(this.templatesDir, 'analysis-templates'))
.map((file) => path.parse(file).name);
const existingTemplates = await this.listTemplates(awsAccountId, orgId);
for (const file of files) {
const templateId = this.cliShared.getTemplateId(orgId, file);
const existing = existingTemplates.find(
template => template.TemplateId === this.cliShared.getTemplateId(orgId, file)
);
if (existing) {
console.log(`[*] template already exists ${templateId}`);
continue;
}
try {
await this.createTemplate(path.basename(file), orgId, awsAccountId, group, superUserArns);
} catch (error) {
if ((error as AWSError).code !== 'ResourceExistsException') {
throw error;
}
console.log(`[*] template already exists ${templateId}`);
}
}
}
public async listTemplates(awsAccountId: string, orgId: string) {
let next: string | undefined;
const ret = [];
do {
const res = await this.awsQuicksight.listTemplates({
AwsAccountId: awsAccountId,
NextToken: next
}).promise();
next = res.NextToken;
ret.push(...(res.TemplateSummaryList || []));
} while (next);
return ret.filter(template => template.TemplateId?.toLowerCase().startsWith(orgId.toLowerCase()));
}
public async createTemplate(
file: string,
orgId: string,
awsAccountId: string,
group: QuickSight.Group,
superUserArns: string[]): Promise<void> {
const templateId = this.cliShared.getTemplateId(orgId, file);
console.log(`[*] creating template ${templateId}`);
const template = fs.readFileSync(
path.join(this.templatesDir, 'analysis-templates', `${file}.json`),
).toString();
const templateJson = JSON.parse(mustache.render(template, {
AwsAccountId: awsAccountId,
TemplateId: templateId,
TemplateName: `${orgId} ${file}`,
}));
await this.awsQuicksight.createTemplate({
...templateJson,
Permissions: [...superUserArns, group.Arn].map(arn => ({
Principal: arn,
Actions: ['quicksight:UpdateTemplatePermissions', 'quicksight:DescribeTemplate'],
}))
}).promise();
await this.waitForTemplateCreation(awsAccountId, templateId);
console.log(`[*] created template ${templateId}`);
}
// this method returns a variable of type promise
// tslint:disable-next-line: promise-function-async
private waitForTemplateCreation(awsAccountId: string, templateId: string): Promise<void> {
let status: string | undefined;
return new Promise(async (res) => {
do {
const templ = await this.awsQuicksight.describeTemplate({
TemplateId: templateId,
AwsAccountId: awsAccountId,
}).promise();
status = templ.Template?.Version?.Status;
await new Promise((resolve) => setTimeout(resolve, 1000));
console.log(`[*] waiting for template : ${templateId} creation : ${templ.Template?.Version?.Status}`);
} while (status === 'CREATION_IN_PROGRESS');
res.call(this);
});
}
}
import { AWSError, QuickSight, STS } from 'aws-sdk';
import { CliConfig } from '../cli-config.model';
import { CliShared } from '../shared.models';
export class CliQuicksightUsers {
private readonly awsSts = new STS();
private awsQuicksight = new QuickSight();
constructor(private readonly config: CliConfig, private readonly cliShared: CliShared) {}
public async getUserArns(users: string[]): Promise<string[]> {
const ret: string[] = [];
let nextToken: string | undefined;
do {
const res = await this.awsQuicksight
.listUsers({
AwsAccountId: await this.cliShared.getAccountId(),
Namespace: 'default',
})
.promise();
nextToken = res.NextToken;
const filteredUsers = res.UserList?.filter(user => user.UserName && users.includes(user.UserName))
.map(user => user.Arn as string)
.filter(arn => !!arn);
ret.push(...(filteredUsers || []));
} while (nextToken);
return ret;
}
public async createUser(email: string, dashboardAccessRoleArn: string): Promise<QuickSight.User | null> {
const awsAccountId = await this.cliShared.getAccountId();
const role = await this.awsSts
.assumeRole({
RoleArn: dashboardAccessRoleArn,
RoleSessionName: email,
})
.promise();
const awsQuicksightLocal = new QuickSight({
credentials: {
accessKeyId: role.Credentials?.AccessKeyId as string,
secretAccessKey: role.Credentials?.SecretAccessKey as string,
sessionToken: role.Credentials?.SessionToken as string,
expireTime: role.Credentials?.Expiration as Date,
},
});
try {
const res = await awsQuicksightLocal
.registerUser({
AwsAccountId: awsAccountId,
Email: email,
IdentityType: 'IAM',
Namespace: 'default',
UserRole: 'READER',
SessionName: email,
IamArn: dashboardAccessRoleArn,
})
.promise();
return res.User as QuickSight.User;
} catch (err) {
if ((err as AWSError).code !== 'ResourceExistsException') {
throw err;
}
console.log('[*] user already exists');
return null;
}
}
public async addUserToGroup(username: string, groupName: string): Promise<void> {
const awsAccountId = await this.cliShared.getAccountId();
await this.awsQuicksight
.createGroupMembership({
AwsAccountId: awsAccountId,
GroupName: groupName,
MemberName: username,
Namespace: 'default',
})
.promise();
}
public async createGroup(orgId: string = this.config.organization.orgId): Promise<QuickSight.Group> {
const awsAccountId = await this.cliShared.getAccountId();
const groupName = `${orgId}Group`;
try {
const res = await this.awsQuicksight
.createGroup({
AwsAccountId: awsAccountId,
Namespace: 'default',
GroupName: groupName,
})
.promise();
return res.Group as QuickSight.Group;
} catch (err) {
if ((err as AWSError).code === 'ResourceExistsException') {
console.log(`[*] group : ${groupName} already exists `);
return await this.getGroup(groupName, awsAccountId);
}
throw err;
}
}
public async getGroup(groupName: string, awsAccountId: string): Promise<QuickSight.Group> {
const res = await this.awsQuicksight
.describeGroup({
AwsAccountId: awsAccountId,
GroupName: groupName,
Namespace: 'default',
})
.promise();
return res.Group as QuickSight.Group;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment