Script to scrape IAM Actions per service directly from the documentation

aws-iam-actions-scraper.go

package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"path"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

const ReferenceURL = "https://docs.aws.amazon.com/service-authorization/latest/reference/reference_policies_actions-resources-contextkeys.html"

type ResourceType struct {
	Name     string
	Required bool
}

type IAMAction struct {
	ActionName    string
	Description   string
	AccessLevel   string
	ResourceTypes []ResourceType
	ConditionKeys []string
}

func getActionsFromPage(page string) (string, []IAMAction) {
	doc, err := goquery.NewDocument(page)
	if err != nil {
		panic(err)
	}

	servicePrefix := ""
	actions := []IAMAction{}

	doc.Find("#main-col-body p code.code").Each(func(i int, s *goquery.Selection) {
		if i == 0 {
			servicePrefix = s.Text()
		}
	})

	doc.Find(".table-container .table-contents table").Each(func(i int, s *goquery.Selection) {
		if i == 0 {
			var action *IAMAction = nil

			s.Find("tr").Each(func(i int, s *goquery.Selection) {
				cells := s.Find("td")

				if cells.Length() > 4 {
					if action != nil {
						actions = append(actions, *action)
					}

					action = &IAMAction{ConditionKeys: []string{}}
				}

				cell := cells.First()

				if cells.Length() > 3 {
					action.ActionName = fmt.Sprintf("%s:%s", servicePrefix, strings.TrimSpace(cell.Text()))

					cell = cell.Next()
					action.Description = strings.TrimSpace(cell.Text())

					cell = cell.Next()
					action.AccessLevel = strings.TrimSpace(cell.Text())

					cell = cell.Next()
				}

				cell.Find("p a").Each(func(i int, s *goquery.Selection) {
					name := strings.TrimSpace(s.Text())

					action.ResourceTypes = append(action.ResourceTypes, ResourceType{
						Name:     strings.TrimRight(name, "*"),
						Required: name[len(name)-1:] == "*",
					})
				})

				cell = cell.Next()
				cell.Find("p a").Each(func(i int, s *goquery.Selection) {
					action.ConditionKeys = append(action.ConditionKeys, strings.TrimSpace(s.Text()))
				})
			})

			if action != nil {
				actions = append(actions, *action)
			}
		}
	})

	return servicePrefix, actions
}

type GroupURL struct {
	Title string
	URL   string
}

func getActionGroupURLs() ([]GroupURL, error) {
	doc, err := goquery.NewDocument(ReferenceURL)
	if err != nil {
		panic(err)
	}

	baseUrl, err := url.Parse(ReferenceURL)
	if err != nil {
		return nil, err
	}

	urls := []GroupURL{}

	doc.Find("div.highlights ul li a").Each(func(i int, s *goquery.Selection) {
		title := strings.TrimSpace(s.Text())
		href, exists := s.Attr("href")

		if exists {
			url, err := baseUrl.Parse(href)
			if err != nil {
				return
			}

			urls = append(urls, GroupURL{Title: title, URL: url.String()})
		}
	})

	return urls, nil
}

func main() {
	var outputDir string
	flag.StringVar(&outputDir, "dir", "./iam-actions", "output directory")

	urls, err := getActionGroupURLs()
	if err != nil {
		panic(err)
	}

	os.MkdirAll(outputDir, 0755)

	for _, url := range urls {
		log.Printf("Scraping actions for %s", url.Title)
		prefix, actions := getActionsFromPage(url.URL)

		output, err := json.MarshalIndent(actions, "", "  ")
		if err != nil {
			panic(err)
		}

		err = os.WriteFile(path.Join(outputDir, fmt.Sprintf("%s.json", prefix)), output, 0644)
		if err != nil {
			panic(err)
		}
	}
}