Eliezer Yudkowsky's non-sequence posts on LessWrong

eliezer_non_sequence_posts.csv

scrape.py

#!/usr/bin/env python3

import requests
import json

query = """
    {
      posts(input: {
        terms: {
          view: "userPosts"
          userId: "nmk3nLpQE89dMRzzN"
          limit: 100
          meta: null  # this seems to get both meta and non-meta posts
          offset: %s
        }
      }) {
        results {
          _id
          title
          pageUrl
          postedAt
          canonicalSequence {
            title
          }
          voteCount
          baseScore
          slug
        }
      }
    }
    """

has_more = True
offset = 0
while has_more:
    r = requests.get('https://www.lesswrong.com/graphql', params={'query': query % offset})
    if r.json()['data']['posts']['results']:
        print(json.dumps(r.json()['data']['posts']['results'], indent=4))
    else:
        has_more = False
    offset += 100


# edit the file manually to combine into a single list
# Then run the following:
# >>> import pandas as pd
# >>> df = pd.read_json('eliezer_dump.json')
# >>> df[df.canonicalSequence.isnull()][['postedAt', 'baseScore', 'title', 'pageUrl']].to_csv('test.csv', index=False)