Resource Contract Conversion.ipynb

{
 "worksheets": [
  {
   "cells": [
    {
     "metadata": {},
     "cell_type": "markdown",
     "source": "# Introduction\n\nThis notebook presents a text-mapping of a document from [ResourceContracts.org](http://www.resourcecontracts.org) into the [Open Contracting Data Standard](http://standard.open-contracting.org) as part of research into how far the data model can capture concessession contracts.\n\nFor more information [see here](https://gist.github.com/practicalparticipation/bb0c1a4acb2888671ec9). \n\n\nInitial work. Last updated 7th December 2014\n"
    },
    {
     "metadata": {},
     "cell_type": "code",
     "input": "from datetime import datetime, date, time\n\n## Function to create a data package\ndef new_package():\n    ocds = {}\n    d = datetime.now()\n    ocds['uri'] = \"http://localhost/tmp\"\n    ocds['publishedDate'] = d.isoformat()\n    ocds['publisher'] = \"tim@practicalparticipation.co.uk - Demonstration\"\n    ocds['license'] = \"Copyright held by original data providers.\"\n    ocds['releases'] = []\n    return ocds",
     "prompt_number": 152,
     "outputs": [],
     "language": "python",
     "trusted": true,
     "collapsed": false
    },
    {
     "metadata": {},
     "cell_type": "code",
     "input": "## Function to search for a text value in ResourceContracts annotations - using an English Language key. \n## Allows just the first few words to be used, and returns the first match. \ndef fetch_annotation(data,search):\n    for annotation in data['document']['annotations']:\n        key = str(annotation['title'].split(\"//\")[-1].lower())\n        if(key.startswith(search.lower())):\n            return annotation['content'].split(\"--\")[0]\n\n    return \"\"\n        ",
     "prompt_number": 143,
     "outputs": [],
     "language": "python",
     "trusted": true,
     "collapsed": false
    },
    {
     "metadata": {},
     "cell_type": "code",
     "input": "## Main script to generate \nimport json\nimport urllib2\nimport uuid\nfrom dateutil.parser import parse\nfrom datetime import datetime, date, time, timedelta\nfrom iso3166 import countries\n\nf = urllib2.urlopen(\"https://www.documentcloud.org/api/documents/439271-the-shallow-water-tano-contract-area.json\" )\ndata = json.loads(f.read())\n\npackage = new_package()\nrelease = {}\nocid = str(uuid.uuid1())\n\nrelease.update({\"ocid\":ocid})\nrelease.update({\"id\":ocid+\"/1\"})\nrelease.update({\"date\":parse(data['document']['updated_at']).isoformat()})\n\nrelease.update({\"language\":data['document']['language']}) #ToDo: at the moment the incoming document uses ISO3. We want ISO2 digit\n\n## Let's have a custom implementation type for now\nrelease.update({\"initiationType\":\"concession\"})\n\n## ToDo: We need to work out what what tag to apply to this release\n\n## Setup planning, tender, award and release elements\nrelease.update({\"planning\":{},\"tender\":{}, \"award\":[],\"contract\":[]})\n\n## AWARD DETAILS\n## We have basic details of the award / contract in the data block\naward = {\"id\": ocid+\"/1/award\",\"title\":data['document']['title']}\ncontract = {\"id\":ocid+\"/1/contract\",\"awardID\":ocid+\"/1/award\",\"title\":data['document']['title']}\n\n## Details of the signature added to the contract\ntry:\n    contract.update({\"dateSigned\":parse(data['document']['data']['Signature Date']).isoformat()})\nexcept:\n    pass\n\n## Assuming the duration is to be calculated from signature date, for the number of years indicated in the meta-data\ntry:\n    start_date = parse(data['document']['data']['Signature Date'])\n    end_date = start_date.replace(year = start_date.year + int(data['document']['data']['Contract Term (duration) in Years']))\n    contract.update({\"period\":{\"startDate\":start_date.isoformat(),\"endDate\":end_date.isoformat()}})\nexcept:\n    pass\n\n## We don't currently have a place to put the country value in OCDS.\n## Attaching this to the top-level for now\n## We need to better identify the semantics of Country (i.e. jurisdiction? primary country of activity?)\ntry:\n    release.update({\"location\":[{\"note\":\"This is a prototype location block, and not currently part of OCDS standard\"}]})\n    country_list = data['document']['data']['Countries'].split(\",\")\n    for country in country_list:\n        release['location'].append({\"schema\":\"ISO3166-1\",\"id\":countries.get(str(country)).alpha2,\"description\":country})\nexcept:\n    pass\n        \n### We'll model the details of the resource covered by the contract within the items block of the award. \n### We need to add an extra element to the item level to indicate the kind of arrangement that relates to this item\nitem = {}\ntry:\n    resource = data['document']['data']['Resource']\n    item_type = fetch_annotation(data,\"Type of document / right\")\n    item['type'] = data['document']['data']['Type of Mining Title']\n    item['description']=fetch_annotation(data,\"type of resources\")\n    item['classification'] = {\"scheme\":\"extractives (custom)\",\"id\":resource,\"description\":item_type}\n    \n    item['location'] = []\n    ### See if we can attach location information to the item\n    if(fetch_annotation(data,\"Name and/or number of field\")):\n        item['location'].append({\"scheme\":\"blocks\",\"id\":\"-\",\"description\":fetch_annotation(data,\"Name and/or number of field\")})\n    \n    if(fetch_annotation(data,\"Location, longitude and latitude\")):\n        item['location'].append({\"scheme\":\"locationDescription\",\"id\":\"-\",\"description\":fetch_annotation(data,\"Location, longitude and latitude\")})\n\n    \nexcept:\n    pass\n\naward.update({\"items\":[item]})\n\n###Buyer / contractingParty\nrelease.update({\"buyer\":{\"name\":fetch_annotation(data,\"State agency\")}})\n\n### Supplier\nsuppliers = []\nsuppliers.append({\"name\":fetch_annotation(data,\"Name of company executing\")})\n\naward.update({\"suppliers\":suppliers})\n\n### And attaching documents... to the contract section?\ncontract.update({\"documents\":[{\"id\":data['document']['id'],\"documentType\":\"contractSigned\",\"title\":data['document']['title'],\"url\":data['document']['canonical_url'],\"datePublished\":parse(data['document']['created_at']).isoformat(),\"dateModified\":parse(data['document']['updated_at']).isoformat(),\"format\":\"text/html\"}]})\n\n\n## Add the awards and contract to the release, and the release to the package\nrelease['award'].append(award)\nrelease['contract'].append(contract)\npackage['releases'].append(release)\n\n## Output it all\nprint json.dumps(package,indent=True)\n\n",
     "prompt_number": 145,
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": "{\n \"publisher\": \"tim@practicalparticipation.co.uk - Demonstration\", \n \"releases\": [\n  {\n   \"language\": \"eng\", \n   \"buyer\": {\n    \"name\": \"Government of the Republic of Ghana, represented by the Minister for Energy, and Ghana National Petroleum Corporation (public corporation)\"\n   }, \n   \"ocid\": \"d9351682-7e7b-11e4-b323-9494260a0f06\", \n   \"contract\": [\n    {\n     \"documents\": [\n      {\n       \"title\": \"Tullow Sabre - Ghana, 2006\", \n       \"url\": \"http://www.documentcloud.org/documents/439271-the-shallow-water-tano-contract-area.html\", \n       \"format\": \"text/html\", \n       \"datePublished\": \"2012-09-24T14:43:07+00:00\", \n       \"documentType\": \"contractSigned\", \n       \"dateModified\": \"2014-11-25T13:08:30+00:00\", \n       \"id\": \"439271-the-shallow-water-tano-contract-area\"\n      }\n     ], \n     \"title\": \"Tullow Sabre - Ghana, 2006\", \n     \"awardID\": \"d9351682-7e7b-11e4-b323-9494260a0f06/1/award\", \n     \"period\": {\n      \"startDate\": \"2006-07-04T00:00:00\", \n      \"endDate\": \"2036-07-04T00:00:00\"\n     }, \n     \"dateSigned\": \"2006-07-04T00:00:00\", \n     \"id\": \"d9351682-7e7b-11e4-b323-9494260a0f06/1/contract\"\n    }\n   ], \n   \"initiationType\": \"concession\", \n   \"tender\": {}, \n   \"planning\": {}, \n   \"award\": [\n    {\n     \"items\": [\n      {\n       \"type\": \"Concession\", \n       \"description\": \"Crude oil (hydrocarbons which are liquid at 14.65 psia pressure and sixty degrees Farenheint, and includes condensates and distillates obtained from natural gas) and natural gas (including wet gas, dry gas and residue gas remaining after the the extraction of liquid hydrocarbons from wet gas)\", \n       \"classification\": {\n        \"scheme\": \"extractives (custom)\", \n        \"id\": \"Hydrocarbons\", \n        \"description\": \"Concession (Petroleum Agreement)\"\n       }, \n       \"location\": [\n        {\n         \"scheme\": \"blocks\", \n         \"id\": \"-\", \n         \"description\": \"North, South and West Tano fields\"\n        }, \n        {\n         \"scheme\": \"locationDescription\", \n         \"id\": \"-\", \n         \"description\": \"Offshore, shallow water blocks comprising an area of 983 square kilometers. The contract area is designated by the coordinates set out in page 2 of Annex 1 to the agreement.\"\n        }\n       ]\n      }\n     ], \n     \"suppliers\": [\n      {\n       \"name\": \"Tullow Ghana Limited, a Jersey company, and Sabre Oiland Gas Limited, a UK company (jointly referred to as \\\"Contractor\\\")\"\n      }\n     ], \n     \"id\": \"d9351682-7e7b-11e4-b323-9494260a0f06/1/award\", \n     \"title\": \"Tullow Sabre - Ghana, 2006\"\n    }\n   ], \n   \"date\": \"2014-11-25T13:08:30+00:00\", \n   \"id\": \"d9351682-7e7b-11e4-b323-9494260a0f06/1\", \n   \"location\": [\n    {\n     \"note\": \"This is a prototype location block, and not currently part of OCDS standard\"\n    }, \n    {\n     \"description\": \"Ghana\", \n     \"id\": \"GH\", \n     \"schema\": \"ISO3166-1\"\n    }\n   ]\n  }\n ], \n \"uri\": \"http://localhost/tmp/\", \n \"license\": \"Copyright held by original data providers.\", \n \"publishedDate\": \"2014-12-07T20:45:11.816815\"\n}\n"
      }
     ],
     "language": "python",
     "trusted": true,
     "collapsed": false
    }
   ],
   "metadata": {}
  }
 ],
 "metadata": {
  "name": "",
  "signature": "sha256:90328f9c4418b1ebd3a36182771d7e6e95de64bd7d2275e8d50acf5552557f81",
  "gist_id": "5c9001db571c9f30b8bf"
 },
 "nbformat": 3
}