Using qualifications to select or omit participants in AMT

qualification-based-HIT-scrubbed.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import datetime\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "from boto.mturk.connection import MTurkConnection, MTurkRequestError\n",
    "from boto.mturk.question import ExternalQuestion\n",
    "from boto.mturk.qualification import LocaleRequirement, PercentAssignmentsApprovedRequirement, \\\n",
    "    Qualifications, NumberHitsApprovedRequirement, Requirement"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is the code for creating a dummy HIT for the purposes of paying someone money through AMT without requiring them to complete a HIT. This HIT will only be veiwable to those whose workerIds you have approved. I've modified some previously used code to create helpful json log files.\n",
    "\n",
    "The sequence is as follows:\n",
    "\n",
    "1. Create a new qualification type. Make sure it does NOT auto-grant. Take note of the qualificationTypeId (this should also be logged now in \"log/qual\")\n",
    "2. Test the one-click HIT page at https://nickrsearcy.github.io/cause-affect-static/one.html\n",
    "3. Create a HIT with that page as an external question. This hit should have a negligible approval delay (1s is a good value) so that once completed it is auto-approved quickly. It should stay up for a long period of time. Make sure that it includes your custom qualification as a requirement and take note of the value you require. 1 is a good value. Make sure that required_to_preview is set to True as it defaults to False. Take note of the HITTypeId of this new HIT.\n",
    "4. Given a list of workerIds, grant each of them the qualification from step 1 and make sure that the value of the qualification matches what you set in step 3.\n",
    "5. Message each worker with a link to the HIT."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# test list\n",
    "list_of_workers = [\n",
    "    \"A2L6IF22AC260S\",\n",
    "    \"ABYGLITD9LAX5\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# real list\n",
    "real_list_of_workers = set(\"\"\"A3HBXVW923NG8Q\n",
    "A17HNBZF5A1CWF\n",
    "A1ICEHLHW6JNN1\n",
    "A1DS5O8MSI3ZH0\n",
    "A3TIUBKMKNPIFL\n",
    "AZACNY1H74YWS\n",
    "A2YS6IE0ZLKJ1P\n",
    "A1207N7QFOFPKB\n",
    "A2O2O2X7PGR8XW\n",
    "ADKKDK6P4MS1Z\n",
    "A15F2JTVAJXITI\n",
    "A2SNMMSKX224GR\n",
    "A382OV019QLW6H\n",
    "A1OUUZSD5DD4ZN\n",
    "A25L985XCNESXE\n",
    "A1UIV4HPLE5JOY\n",
    "A9BRH899F1PW7\n",
    "A42UZIPIKDFI5\n",
    "A3EN54EJAR9QP0\n",
    "AZNNPGW3VNPQ2\n",
    "A3PUU1XU4C1A3V\n",
    "A15LHHN76OW2UM\n",
    "AF18OIZ0GWGP2\n",
    "AK0PU96DYAJXN\n",
    "A2I9PUZT19ZBO8\n",
    "A3QRZPJT2CT2IK\n",
    "A26JRMEO37XT6Z\n",
    "A3V3GOQYTMMO5S\n",
    "A3R0QO6884CV1P\n",
    "AO24K1MDNQ0YX\n",
    "A20ASMCESA51U4\n",
    "A2G8UI423QZYJJ\n",
    "AS8VFYAHXDBAK\n",
    "A3D38UO96N9SB0\n",
    "A1WE7JOOWZSMHF\n",
    "A1TLNLB9D87H6\n",
    "A1FYS4T8BCR1AT\n",
    "ATADQXPHL10Y8\n",
    "A1CA1PH52XPXPR\n",
    "A1R0689JPSQ3OF\n",
    "AL44289RBKXUT\n",
    "ATADQXPHL10Y8\n",
    "AJSOA5J0L6I3C\n",
    "A1CPVTZAK27YSF\n",
    "A1IFIK8J49WBER\n",
    "A3I3JQX0P2F0K3\n",
    "A27BKFN6UB1LDP\n",
    "A2QT68NJYAJV9P\n",
    "A2VD133723IZR8\n",
    "A1VPE05UDNFKGV\n",
    "A3K7UC7LUKV0ZV\n",
    "A2VNR6984SDFGQ\n",
    "A3862RIFFUV141\n",
    "A34L4JED8UAIXC\n",
    "A324FQ64NCN2YS\n",
    "A33OQJ5NRKXQU3\n",
    "A3F8G26PENQ6EO\n",
    "A2US16825HDCK7\n",
    "A3IQAKT74IXJB6\n",
    "A2JCK494NV7TFX\n",
    "A39W3PYK82IBAS\n",
    "A35W13N45CZB1O\n",
    "A3CIUPLZ6614U2\n",
    "ABE0TGK79GGLA\n",
    "A36KDWI1CGJFFA\n",
    "A2C39KTRMOM1XZ\n",
    "A2ABSQBRUAUS4C\n",
    "A3DRYZ95VDUDK\n",
    "A1LRJ2MQD4AMES\n",
    "A1K75ALN5NQIJM\n",
    "A3MK8WQ4S3WUC2\n",
    "ABJGKVPL5WF80\n",
    "A3MMOZDMBZ8K42\n",
    "A3MELYYGRJ61SX\n",
    "A30QRYULPVYZQV\n",
    "A3GO51V2J3BBCT\n",
    "A1B5O1E2T429ET\n",
    "A1NGJWRRI9NDED\n",
    "A1SVSCF7JWBIJL\n",
    "A3NUIEPMZSHEZ4\n",
    "A28RX7L0QZ993M\n",
    "A2SMROLJ39OR74\n",
    "A2HOUSKURBF8UA\n",
    "AJALV8WOILDPA\n",
    "A3MHYBS6PHJ5QG\n",
    "A1I9HRO8HAVLWS\n",
    "A39YX0NP6W8TSQ\n",
    "A2XPE8E9H3D8AZ\n",
    "A3TP3QZ1VI7SHX\n",
    "A3DIRN48SZVNRE\n",
    "AX0KHOWMMM6Z2\n",
    "A38JZ9Q0S0ZPO8\n",
    "A31Y2UA2LPDSWZ\n",
    "A1PYODCHRF96S8\n",
    "ARXKQFSEKTYID\n",
    "A2DNWDHMMBSHWT\n",
    "A2NIG9U9WU0AFU\n",
    "A2W8ZV9DQWVHJK\n",
    "A146450W3WOMQ2\n",
    "A3UF62O23KAZBW\n",
    "ASTDBTVY3WP1K\n",
    "A36O6HV8LCXLJR\n",
    "AZOR3BXT0WZQS\n",
    "A2XX4E9CDNHX5\n",
    "A3UF62O23KAZBW\n",
    "A1ISWXN9VA49R8\n",
    "A1ZVSHYKYCIPNZ\n",
    "A3M0HR7RHCWZM1\n",
    "A1ZVSHYKYCIPNZ\n",
    "A13KYLCBVG5BEB\n",
    "A10P1S4IDALVR1\n",
    "A1GVHFB9F270GX\n",
    "A3M7LLWCEAZYSN\n",
    "ASTR3EPUOKEXV\"\"\".split(\"\\n\"))\n",
    "# uncomment the following line for production\n",
    "# list_of_workers = list(real_list_of_workers) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Nick (workerId A2L6IF22AC260S)\n",
    "# AK = \"blah\"\n",
    "# SK = \"blah\"\n",
    "\n",
    "# Cocosci (workerid ABYGLITD9LAX5)\n",
    "AK = \"blah\"\n",
    "SK = \"blah\"\n",
    "\n",
    "# HOST = \"mechanicalturk.sandbox.amazonaws.com\"\n",
    "HOST = \"mechanicalturk.amazonaws.com\"\n",
    "EXTERNAL_URL = \"https://nickrsearcy.github.io/cause-affect-static/one.html\"\n",
    "MAX_ASSIGNMENTS = len(list_of_workers)\n",
    "\n",
    "mtc = MTurkConnection(\n",
    "  aws_access_key_id=AK, \n",
    "  aws_secret_access_key=SK,\n",
    "  host=HOST,\n",
    "#  profile_name=\"default\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_qualification_type(mtc, logdir=\"log/qual\", **kwargs):\n",
    "    r = mtc.create_qualification_type(**kwargs)\n",
    "    q = r[0]\n",
    "    logdict = kwargs\n",
    "    logdict[\"AutoGranted\"] = q.AutoGranted\n",
    "    logdict[\"CreationTime\"] = q.CreationTime\n",
    "    logdict[\"IsValid\"] = q.IsValid\n",
    "    logdict[\"Description\"] = q.Description\n",
    "    logdict[\"IsValid\"] = q.IsValid\n",
    "    logdict[\"Name\"] = q.Name\n",
    "    logdict[\"QualificationType\"] = q.QualificationType\n",
    "    logdict[\"QualificationTypeId\"] = q.QualificationTypeId\n",
    "    logdict[\"QualificationTypeStatus\"] = q.QualificationTypeStatus\n",
    "    logdict[\"Time\"] = datetime.datetime.utcnow().ctime()\n",
    "    logfile = str(int(time.time()))+\".json\"\n",
    "    if not os.path.isdir(logdir):\n",
    "        os.makedirs(logdir)\n",
    "    logpath = os.path.join(logdir,logfile)\n",
    "    with open(logpath,'a') as f:\n",
    "        json.dump(logdict,f)\n",
    "    print(logdict)\n",
    "    return logdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def create_hit(mtc, logdir=\"log/hit\", **kwargs):\n",
    "    r = mtc.create_hit(**kwargs)\n",
    "    h = r[0]\n",
    "    logdict = kwargs\n",
    "    logdict[\"HITId\"] = h.HITId\n",
    "    logdict[\"HITTypeId\"] = h.HITTypeId\n",
    "    logdict[\"IsValid\"] = h.IsValid\n",
    "    logdict[\"Time\"] = datetime.datetime.utcnow().ctime()\n",
    "    logdict[\"qualifications\"] = kwargs[\"qualifications\"].get_as_params()\n",
    "    logdict[\"question\"] = kwargs[\"question\"].get_as_params()\n",
    "    logfile = str(int(time.time()))+\".json\"\n",
    "    if not os.path.isdir(logdir):\n",
    "        os.makedirs(logdir)\n",
    "    logpath = os.path.join(logdir,logfile)\n",
    "    with open(logpath,'a') as f:\n",
    "        json.dump(logdict,f)\n",
    "    print(logdict)\n",
    "    return logdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'description': '\\nThis qualification is designed to allow access to a one-time HIT\\nto allow easy compensation.', 'auto_granted_value': 1, 'QualificationTypeId': '3U7W11ZQ9O50JGTBE7UPR4A8RQHJVZ', 'Name': 'Compensation HIT Wed Apr 20 13:35:19 2016', 'QualificationType': '', 'keywords': None, 'test_duration': None, 'QualificationTypeStatus': 'Active', 'status': 'Active', 'answer_key': None, 'retry_delay': None, 'IsValid': 'True', 'auto_granted': False, 'test': None, 'AutoGranted': '0', 'CreationTime': '2016-04-20T13:35:19Z', 'Description': '\\nThis qualification is designed to allow access to a one-time HIT\\nto allow easy compensation.', 'name': 'Compensation HIT Wed Apr 20 13:35:19 2016', 'Time': 'Wed Apr 20 13:35:19 2016', 'answer_key_xml': None}\n"
     ]
    }
   ],
   "source": [
    "# boto does not gracefully fail if you try to create a qualification with the same name as one that already exists.\n",
    "# instead of catching and managing the error message, ensure names are unique by adding the time.\n",
    "qual_name = \"Compensation HIT \" + datetime.datetime.utcnow().ctime() \n",
    "comp_qual = create_qualification_type(mtc,\n",
    "    name=qual_name, \n",
    "    description=\"\"\"\n",
    "This qualification is designed to allow access to a one-time HIT\n",
    "to allow easy compensation.\"\"\", \n",
    "    status=\"Active\", \n",
    "    keywords=None, \n",
    "    retry_delay=None, \n",
    "    test=None, \n",
    "    answer_key=None, \n",
    "    answer_key_xml=None, \n",
    "    test_duration=None, \n",
    "    auto_granted=False, \n",
    "    auto_granted_value=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'description': 'Test.', 'HITTypeId': '37BTIVOV5AVD68LAIJ4GX07J793JK2', 'questions': None, 'keywords': 'test', 'title': 'Test.', 'qualifications': {'QualificationRequirement.1.IntegerValue': 2, 'QualificationRequirement.1.Comparator': 'GreaterThanOrEqualTo', 'QualificationRequirement.1.RequiredToPreview': 'true', 'QualificationRequirement.1.QualificationTypeId': '3U7W11ZQ9O50JGTBE7UPR4A8RQHJVZ'}, 'lifetime': 172800, 'approval_delay': 1, 'reward': 0.75, 'max_assignments': 1, 'HITId': '3ZXNP4Z39RQBGSU78HL3PUM25D67LI', 'IsValid': 'True', 'hit_type': None, 'duration': 600, 'question': {'ExternalQuestion': '<ExternalQuestion xmlns=\"http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd\"><ExternalURL>https://nickrsearcy.github.io/cause-affect-static/one.html</ExternalURL><FrameHeight>600</FrameHeight></ExternalQuestion>'}, 'Time': 'Wed Apr 20 13:35:21 2016'}\n"
     ]
    }
   ],
   "source": [
    "quals = Qualifications();\n",
    "quals.add( Requirement(comp_qual[\"QualificationTypeId\"], 'GreaterThanOrEqualTo',2, required_to_preview=True) )\n",
    "sorry_hit = create_hit(mtc,\n",
    "  hit_type=None,\n",
    "  question = ExternalQuestion(EXTERNAL_URL, 600),\n",
    "  lifetime = 48*60*60, # Amount of time HIT will be available to accept unless 'max_assignments' are accepted before\n",
    "  max_assignments = MAX_ASSIGNMENTS,\n",
    "  title = 'Test.',\n",
    "  description = 'Test.',\n",
    "  keywords = 'test',\n",
    "  reward = 0.75,\n",
    "  duration = 10*60, # Maximum amount of time turkers are allowed to spend on HIT\n",
    "  approval_delay = 1, # Amount of time after which HIT is automatically approved\n",
    "  questions = None,\n",
    "  qualifications = quals )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A2SNMMSKX224GR\n"
     ]
    }
   ],
   "source": [
    "# you get an error if you do this twice, might have to try-catch\n",
    "for wid in list_of_workers:\n",
    "    response = mtc.assign_qualification(comp_qual[\"QualificationTypeId\"],wid,value=2,send_notification=False)\n",
    "    print(wid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "if HOST == \"mechanicalturk.sandbox.amazonaws.com\":\n",
    "    base_url = \"https://workersandbox.mturk.com/mturk/preview?groupId=\"\n",
    "else:\n",
    "    base_url = \"https://www.mturk.com/mturk/preview?groupId=\"\n",
    "HIT_url = base_url + sorry_hit['HITTypeId']\n",
    "message = \"\"\"\n",
    "Hello\n",
    "\n",
    "We have determined that you participated in a faulty HIT hosted by this lab. \n",
    "We allowed workers to complete the entire task but an error prevented anyone \n",
    "from submitting the data at the conclusion, and thus prevented anyone from \n",
    "getting paid for the work they had just done. This qualificition will allow \n",
    "you to participate in a one-time HIT that will provide compensation for the \n",
    "previous faulty HIT. We have already assigned you the qualification \n",
    "necessary for this HIT and you can find it at the link below.\n",
    "\n",
    "{}\n",
    "\n",
    "This HIT will be live for 48 hours. If you are not able to complete it in \n",
    "that time please let us know. We are sorry for any inconvenience.\n",
    "\n",
    "\n",
    "Best\n",
    "CoDaSLab\n",
    "\n",
    "\"\"\".format(HIT_url)\n",
    "subject = \"Compensation for faulty HIT\"\n",
    "for wid in list_of_workers:\n",
    "    mtc.notify_workers(wid,subject,message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}