POST data checks - config.yaml

rb-config.yaml

# RESTBase config

info:
  name: restbase


templates:

  wmf-content-1.0.0: &wp/content/1.0.0
    swagger: '2.0'
    # swagger options, overriding the shared ones from the merged specs (?)
    info: &wp/content-info/1.0.0
      version: 1.0.0-beta
      title: Wikimedia REST API
      description: >
          This API aims to provide straightforward and low-latency access to
          Wikimedia content and services. It is currently in beta testing, so
          things aren't completely locked down yet. Each entry point has
          explicit stability markers to inform you about development status
          and change policy, according to [our API version
          policy](https://www.mediawiki.org/wiki/API_versioning).

          ### High-volume access
            - As a general rule, don't perform more than 200 requests/s to
              this API.
            - Set a unique `User-Agent` header that allows us to contact you
              quickly. Email addresses or URLs of contact pages work well.
            - Consider using our [HTML
              dumps](https://phabricator.wikimedia.org/T17017) once they
              become available.

      termsOfService: https://wikimediafoundation.org/wiki/Terms_of_Use
      contact:
        name: the Wikimedia Services team
        url: http://mediawiki.org/wiki/RESTBase
      license:
        name: Apache2
        url: http://www.apache.org/licenses/LICENSE-2.0
    x-subspecs:
      - mediawiki/v1/content
      - mediawiki/v1/graphoid
      - mediawiki/v1/mobileapps
      - media/v1/mathoid
    # - mediawiki/v1/revision-scoring
    securityDefinitions: &wp/content-security/1.0.0
      mediawiki_auth:
        description: Checks permissions using MW api
        type: apiKey
        in: header
        name: cookie
        x-internal-request-whitelist:
          - http://en.wikipedia.org/w/api.php
          - http://fr.wikipedia.org/w/api.php
          # Left as a regex for test purpose
          - /http:\/\/parsoid\-lb\.eqiad\.wikimedia\.org/
      header_match:
        description: Checks client ip against one of the predefined whitelists
        x-error-message: This client is not allowed to use the endpoint
        x-whitelists:
          internal:
            - /^(?:::ffff:)?(?:10|127)\./

  wmf-sys-1.0.0: &wp/sys/1.0.0
    info:
      title: Default MediaWiki sys API module
      version: 1.0.0
    paths:
      /{module:table}: &wp/sys/table
        x-modules:
          # There can be multiple modules too per stanza, as long as the
          # exported symbols don't conflict. The operationIds from the spec
          # will be resolved against all of the modules.
          - name: restbase-mod-table-cassandra
            version: 1.0.0
            type: npm
            options: # Passed to the module constructor
              conf:
                hosts: [localhost]
                keyspace: system
                username: cassandra
                password: cassandra
                defaultConsistency: one # or 'one' for single-node testing
                storage_groups:
                  - name: test.group.local
                    domains: /./

      /{module:page_revisions}: &wp/sys/page_revisions
        x-modules:
            - name: page_revisions
              version: 1.0.0
              type: file

      /{module:key_rev_value}: &wp/sys/key_rev_value
        x-modules:
          - name: key_rev_value
            version: 1.0.0
            type: file

      /{module:action}: &wp/sys/action
        x-modules:
          - name: action
            type: file
            options:
              apiRequest:
                method: post
                uri: http://{domain}/w/api.php
                headers:
                  host: '{$.request.params.domain}'
                body: '{$.request.body}'

      /{module:page_save}: &wp/sys/page_save
        x-modules:
          - name: page_save
            type: file

      /{module:parsoid}: &wp/sys/parsoid
        x-modules:
          - name: parsoid
            version: 1.0.0
            type: file
            options:
              parsoidHost: http://parsoid-lb.eqiad.wikimedia.org
              # For local testing, use:
              #parsoidHost: http://localhost:8000

      /{module:graphoid}/v1/png/{title}/{revision}/{graph_id}:
        get:
          x-request-handler:
            - get_from_graphoid:
                request:
                  uri: http://graphoid.wikimedia.org/{domain}/v1/png/{title}/{revision}/{graph_id}

      /{module:mobileapps}:
        x-subspec:
          info:
            title: Mobileapps sys API module
          paths:
            /html/{title}:
              get:
                x-request-handler:
                  - get_from_backend:
                      request:
                        uri: http://appservice.wmflabs.org/{domain}/v1/page/mobile-html/{title}
            /sections/{title}:
              get:
                x-request-handler:
                  - get_from_backend:
                      request:
                        uri: http://appservice.wmflabs.org/{domain}/v1/page/mobile-html-sections/{title}
            /sections-lead/{title}:
              get:
                x-request-handler:
                  - get_from_backend:
                      request:
                        uri: http://appservice.wmflabs.org/{domain}/v1/page/mobile-html-sections-lead/{title}
            /sections-remaining/{title}:
              get:
                x-request-handler:
                  - get_from_backend:
                      request:
                        uri: http://appservice.wmflabs.org/{domain}/v1/page/mobile-html-sections-remaining/{title}
            /text/{title}:
              get:
                x-request-handler:
                  - get_from_backend:
                      request:
                        uri: http://appservice.wmflabs.org/{domain}/v1/page/mobile-text/{title}

#      /{module:revscore}:
#        title: Simple revscore service wrapper
#        x-modules:
#          # Generic revision service interface; Expects requests of the form
#          # /{title}/{revision}.
#          # Specific interface documentation (content types etc) at public
#          # entry point, although we might also want to enforce them
#          # internally.
#          - name: restbase-mod-service
#            version: 1.0.0 # simple service module, to be shared
#            options:
#              storage:
#                uri: /{domain}/sys/key_rev_value/revscore.scores/{title}/{revision}
#              service: 
#                uri: http://revscore.wikimedia.org/{domain}/{title}/{revision}


  global-content: &gb/content/1.0.0
    swagger: '2.0'
    info: *wp/content-info/1.0.0
    x-subspecs:
      - media/v1/mathoid
    #securityDefinitions: *wp/content-security/1.0.0


  global-sys: &gb/sys/1.0.0
    info:
      title: Global domain sys API module
      version: 1.0.0
    paths:
      /{module:table}: *wp/sys/table
      /{module:key_value}: &gb/sys/key_value
        x-modules:
          - name: key_value
            version: 1.0.0
            type: file

      /{module:post_data}: &gb/sys/post_data
        x-modules:
          - name: post_data
            version: 1.0.0
            type: file

      /{module:mathoid}/v1/request/{format}:
        post:
          x-request-handler:
            - store:
                request:
                  method: put
                  uri: /{domain}/sys/post_data/mathoid.request/
                  body:
                    q: '{$.request.body.q}'
                    type: '{$$.default($.request.body.type, "tex")}'
            - handle_req:
                request:
                  method: post
                  uri: /{domain}/sys/mathoid/v1/handler/{format}
                  headers:
                    cache-control: '{cache-control}'
                    x-resource-location: '{$.store.body}'
                  body:
                    q: '{$.request.body.q}'
                    type: '{$$.default($.request.body.type, "tex")}'
      /{module:mathoid}/v1/request/{format}/{hash}:
        get:
          x-setup-handler:
            - init:
                uri: /{domain}/sys/post_data/mathoid.request
          x-request-handler:
            - get_from_storage:
                request:
                  method: get
                  uri: /{domain}/sys/post_data/mathoid.request/{hash}
            - handle_req:
                request:
                  method: post
                  uri: /{domain}/sys/mathoid/v1/handler/{format}
                  headers:
                    cache-control: '{cache-control}'
                    x-resource-location: '{$.request.params.hash}'
                  body: '{$.get_from_storage.body}'
      /{module:mathoid}/v1/handler/{format}:
        post:
          x-setup-handler:
            - init:
                uri: /{domain}/sys/key_value/mathoid.data
                body:
                  keyType: string
                  valueType: json
          x-request-handler:
            - check_storage:
                request:
                  method: get
                  uri: /{domain}/sys/key_value/mathoid.data/{$.request.headers.x-resource-location}
                  headers:
                    cache-control: '{cache-control}'
                return_if:
                  status: '2xx'
                return:
                  status: 200
                  headers: '{$$.merge($.check_storage.body.data[$.request.params.format].headers,$.check_storage.body.headers)}'
                  body: '{$.check_storage.body.data[$.request.params.format].body}'
                catch:
                  status: 404
            - mathoid:
                request:
                  method: post
                  uri: http://mathoid-tester.wmflabs.org/complete
                  #uri: http://localhost:10044/complete
                  body:
                    q: '{$.request.body.q}'
                    type: '{$.request.body.type}'
            - store:
                request:
                  method: put
                  uri: /{domain}/sys/key_value/mathoid.data/{$.request.headers.x-resource-location}
                  headers:
                    content-type: application/json
                  body:
                    headers: '{$$.merge($.mathoid.headers, {"x-resource-location": $.request.headers.x-resource-location})}'
                    data: '{$.mathoid.body}'
                return:
                  status: 200
                  headers: '{$$.merge($.mathoid.body[$.request.params.format].headers,$$.merge($.mathoid.headers, {"x-resource-location": $.request.headers.x-resource-location}))}'
                  body: '{$.mathoid.body[$.request.params.format].body}'


  wp-default-1.0.0: &wp/default/1.0.0
    x-subspecs:
      - paths:
          /{api:v1}:
            x-subspec: *wp/content/1.0.0
      - paths:
          /{api:sys}:
            x-subspec: *wp/sys/1.0.0

  global-default-1.0.0: &gb/default/1.0.0
    x-subspecs:
      - paths:
          /{api:v1}:
            x-subspec: *gb/content/1.0.0
      - paths:
          /{api:sys}:
            x-subspec: *gb/sys/1.0.0


spec: &spec
  title: "The RESTBase root"
  # Some more general RESTBase info
  paths:
    /{domain:en.wikipedia.org}: *wp/default/1.0.0
#    /{domain:he.wikipedia.org}: *wp/default/1.0.0
#    /{domain:de.wikipedia.org}: *wp/default/1.0.0
#   /{domain:es.wikipedia.org}: *wp/default/1.0.0
#   /{domain:nl.wikipedia.org}: *wp/default/1.0.0
#
    # test domains
#    /{domain:en.wikipedia.test.local}: *wp/default/1.0.0
#    /{domain:localhost:8080}: *wp/default/1.0.0

    # global domain
    /{domain:wikimedia.org}: *gb/default/1.0.0


services:
  - name: restbase
    module: ./lib/server
    conf: 
      port: 7231
      spec: *spec
      salt: keko
      default_page_size: 250
      user_agent: RESTBase/wiki-istria

logging:
  name: restbase
  level: debug
  #streams:
  ## XXX: Use gelf-stream -> logstash
  #- type: file
  #  path: /home/doorman/tmp/rb/titi.log
  #  host: <%= @logstash_host %>
  #  port: <%= @logstash_port %>

metrics:
  type: log
  #type: txstatsd
  #host: localhost
  #port: 8125

# Number of worker processes to spawn.
# Set to 0 to run everything in a single process without clustering.
# Use 'ncpu' to run as many workers as there are CPU units
num_workers: 0

# Log error messages and gracefully restart a worker if v8 reports that it
# uses more heap (note: not RSS) than this many mb.
worker_heap_limit_mb: 125