diff --git a/.coveragerc b/.coveragerc index ed8916e6e7..aa1f8546c6 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,5 +2,3 @@ branch = True include = boto3/* -omit = - boto3/docs.py diff --git a/boto3/docs.py b/boto3/docs.py index 48994028d0..87776bf5e0 100644 --- a/boto3/docs.py +++ b/boto3/docs.py @@ -120,16 +120,20 @@ def html_to_rst(html, indent=0, indent_first=False): return rst -def docs_for(service_name): +def docs_for(service_name, session=None, resource_filename=None): """ Generate reference documentation (low and high level) for a service by name. This generates docs for the latest available version. :type service_name: string :param service_name: The service short-name, like 'ec2' + :type session: botocore.session.Session + :param session: Existing pre-setup session or ``None``. :rtype: string """ - session = botocore.session.get_session() + if session is None: + session = botocore.session.get_session() + service_model = session.get_service_model(service_name) print('Processing {0}-{1}'.format(service_name, service_model.api_version)) @@ -151,17 +155,26 @@ def docs_for(service_name): docs += '.. contents:: Table of Contents\n :depth: 2\n\n' - docs += document_client(service_name, official_name, service_model) + # TODO: Get this information from the model somehow in the future. + # For now creating and introspecing a client is a quick and + # dirty way to access waiters/paginators. + client = session.create_client(service_name, aws_access_key_id='dummy', + aws_secret_access_key='dummy', + region_name='us-east-1') + + docs += document_client(service_name, official_name, service_model, + client) docs += document_client_waiter(session, official_name, service_name, - service_model) + service_model, client) - filename = (os.path.dirname(__file__) + '/data/resources/' - '{0}-{1}.resources.json').format(service_name, - service_model.api_version) + if resource_filename is None: + resource_filename = (os.path.dirname(__file__) + '/data/resources/' + '{0}-{1}.resources.json').format( + service_name, service_model.api_version) # We can't use a set here because dicts aren't hashable! models = {} - if os.path.exists(filename): - data = json.load(open(filename)) + if os.path.exists(resource_filename): + data = json.load(open(resource_filename)) model = ResourceModel(service_name, data['service'], data['resources']) for collection_model in model.collections: @@ -208,7 +221,7 @@ def docs_for(service_name): return docs -def document_client(service_name, official_name, service_model): +def document_client(service_name, official_name, service_model, client): """ Generate low-level client documentation for a service. This generates documentation for all available operations. @@ -221,13 +234,6 @@ def document_client(service_name, official_name, service_model): docs += ' {service} = boto3.client(\'{service}\')\n\n'.format( service=service_name) - # TODO: Get this information from the model somehow in the future. - # For now creating and introspecing a client is a quick and - # dirty way to access waiters/paginators. - client = boto3.client(service_name, aws_access_key_id='dummy', - aws_secret_access_key='dummy', - region_name='us-east-1') - wdoc = '' if client.waiter_names: # This gets included in alphabetical order below! @@ -252,10 +258,7 @@ def document_client(service_name, official_name, service_model): return docs def document_client_waiter(session, official_name, service_name, - service_model): - client = boto3.client(service_name, aws_access_key_id='dummy', - aws_secret_access_key='dummy', - region_name='us-east-1') + service_model, client): waiter_spec_doc = '' if client.waiter_names: waiter_spec_doc = 'Waiter\n------\n\n' @@ -556,7 +559,7 @@ def document_operation(operation_model, service_name, operation_name=None, param_desc = ', '.join([ ', '.join(['{0}=None'.format(k) for k in required_params]), ', '.join(['{0}=None'.format(k) for k in optional_params]) - ]) + ]).strip(', ') if operation_name is None: operation_name = xform_name(operation_model.name) @@ -601,7 +604,7 @@ def document_operation(operation_model, service_name, operation_name=None, if param_type in ['list', 'dict']: param_desc = ('\n Structure description::\n\n' + ' ' + key + ' = ' + - document_structure( + document_param_response( key, value, indent=12, indent_first=False) + '\n' + param_desc) required = key in required_params and 'Required' or 'Optional' @@ -615,21 +618,13 @@ def document_operation(operation_model, service_name, operation_name=None, output_shape = operation_model.output_shape if rtype in ['list', 'dict'] and output_shape is not None: docs += (' :return:\n Structure description::\n\n' + - document_structure(None, output_shape, indent=12) + '\n') + document_param_response(None, output_shape, indent=12) + '\n') return docs -PARAM_NAME = "'{name}': " -ELLIPSIS = '...\n' -STRUCT_START = '{\n' -STRUCT_END = '}' -LIST_START = '[\n' -LIST_END = ']' - - -def document_structure(name, shape, indent=0, indent_first=True, - parent_type=None, eol='\n'): +def document_param_response(name, shape, indent=0, indent_first=True, + parent_type=None, eol='\n'): """ Document a nested structure (list or dict) parameter or return value as a snippet of Python code with dummy placeholders. For example: @@ -661,6 +656,7 @@ def document_structure(name, shape, indent=0, indent_first=True, :param eol: The end-of-line string to use when finishing a member. :rtype: string """ + param_name = "'{name}': " docs = '' spaces = ' ' * indent @@ -671,43 +667,100 @@ def document_structure(name, shape, indent=0, indent_first=True, if shape.type_name == 'structure': # Only include the name if the parent is also a structure. if parent_type == 'structure': - docs += PARAM_NAME.format(name=name) - - docs += STRUCT_START - - # Go through each member and recursively process them. - for i, member_name in enumerate(shape.members): - # Individual members get a trailing comma only if they - # are not the last item. - member_eol = '\n' - if i < len(shape.members) - 1: - member_eol = ',' + member_eol - - docs += document_structure( - member_name, shape.members[member_name], - indent=indent + 4, parent_type=shape.type_name, - eol=member_eol) - docs += spaces + STRUCT_END + eol + docs += param_name.format(name=name) + + docs += render_structure(shape, spaces, indent, eol) elif shape.type_name == 'list': # Only include the name if the parent is a structure. if parent_type == 'structure': - docs += PARAM_NAME.format(name=name) + docs += param_name.format(name=name) - docs += LIST_START + docs += render_list(shape, spaces, indent, eol) + elif shape.type_name == 'map': + if parent_type == 'structure': + docs += param_name.format(name=name) - # Lists have only a single member. Here we document it, plus add - # an ellipsis to signify that more of the same member type can be - # added in a list. - docs += document_structure( - None, shape.member, indent=indent + 4, eol=',\n') - docs += spaces + ' ' + ELLIPSIS - docs += spaces + LIST_END + eol + docs += render_map(shape, spaces, indent, eol) else: # It's not a structure or list, so document the type. Here we # try to use the equivalent Python type name for clarity. if name is not None: - docs += PARAM_NAME.format(name=name) + docs += param_name.format(name=name) docs += py_type_name(shape.type_name).upper() + eol return docs + + +def render_structure(shape, spaces, indent, eol): + """ + Render out a ``structure`` type. This renders info about each + member in the shape:: + + { + 'MemberName': 'Value' + } + + """ + docs = '{\n' + + # Go through each member and recursively process them. + for i, member_name in enumerate(shape.members): + # Individual members get a trailing comma only if they + # are not the last item. + member_eol = '\n' + if i < len(shape.members) - 1: + member_eol = ',' + member_eol + + docs += document_param_response( + member_name, shape.members[member_name], + indent=indent + 4, parent_type=shape.type_name, + eol=member_eol) + docs += spaces + '}' + eol + + return docs + + +def render_list(shape, spaces, indent, eol): + """ + Render out a ``list`` type. This renders info about the member + type of the list and adds an ellipsis to indicate it can contain + many items:: + + [ + 'STRING', + ... + ] + + """ + docs = '[\n' + + # Lists have only a single member. Here we document it, plus add + # an ellipsis to signify that more of the same member type can be + # added in a list. + docs += document_param_response( + None, shape.member, indent=indent + 4, eol=',\n') + docs += spaces + ' ...\n' + docs += spaces + ']' + eol + + return docs + + +def render_map(shape, spaces, indent, eol): + """ + Render out a ``structure`` type. This renders info about each + member in the shape:: + + { + 'MemberName': 'Value' + } + + """ + docs = '{\n' + + # Document the types for the keys and values. + docs += (spaces + ' ' + py_type_name(shape.key.type_name).upper() + + ': ' + py_type_name(shape.value.type_name).upper() + '\n') + docs += spaces + '}' + eol + + return docs diff --git a/tests/unit/data/aws/todo/2015-04-01.normal.json b/tests/unit/data/aws/todo/2015-04-01.normal.json new file mode 100644 index 0000000000..1d71b404a3 --- /dev/null +++ b/tests/unit/data/aws/todo/2015-04-01.normal.json @@ -0,0 +1,185 @@ +{ + "metadata":{ + "apiVersion":"2015-04-01", + "endpointPrefix":"todo", + "jsonVersion":"1.1", + "serviceFullName":"AWS ToDo Sample API for Tasks", + "serviceAbbreviation":"AWS ToDo Tasks", + "signatureVersion":"v4", + "protocol":"json" + }, + "documentation":"

AWS sample API that tracks to-do items.

", + "operations":{ + "CreateToDo":{ + "name":"CreateToDo", + "http":{ + "method":"POST", + "requestUri":"/todos" + }, + "input":{ + "shape":"CreateToDoInput", + "documentation":"

Container of the newly created to-do's values.

" + }, + "output":{ + "shape":"ToDoItem", + "documentation":"

A single ToDo item.

" + }, + "errors":[ + { + "shape":"ToDoServerException", + "exception":true, + "documentation":"

A server-side error occurred during the API call. The error message will contain additional details about the cause.

" + }, + { + "shape":"ToDoClientException", + "exception":true, + "documentation":"

The API was called with invalid parameters. The error message will contain additional details about the cause.

" + } + ], + "documentation":"

Create a new to-do item.

" + }, + "DescribeToDos": { + "name":"DescribeToDos", + "http":{ + "method":"GET", + "requestUri":"/todos" + }, + "output":{ + "shape":"ToDoList", + "documentation":"

List of to-do items.

" + }, + "errors":[ + { + "shape":"ToDoServerException", + "exception":true, + "documentation":"

A server-side error occurred during the API call. The error message will contain additional details about the cause.

" + } + ], + "documentation":"

Create a new to-do item.

" + }, + "GetToDo":{ + "name":"GetToDo", + "http":{ + "method":"GET", + "requestUri":"/todos/{Id+}" + }, + "input":{ + "shape":"ToDoInput" + }, + "output":{ + "shape":"ToDoItem" + }, + "errors":[ + { + "shape":"ToDoServerException", + "exception":true, + "documentation":"

A server-side error occurred during the API call. The error message will contain additional details about the cause.

" + }, + { + "shape":"ToDoClientException", + "exception":true, + "documentation":"

The API was called with invalid parameters. The error message will contain additional details about the cause.

" + } + ], + "documentation":"

Get an existing to-do item.

" + }, + "DeleteToDo":{ + "name":"DeleteToDo", + "http":{ + "method":"DELETE", + "requestUri":"/todos/{Id+}" + }, + "input":{ + "shape":"ToDoInput" + }, + "errors":[ + { + "shape":"ToDoServerException", + "exception":true, + "documentation":"

A server-side error occurred during the API call. The error message will contain additional details about the cause.

" + }, + { + "shape":"ToDoClientException", + "exception":true, + "documentation":"

The API was called with invalid parameters. The error message will contain additional details about the cause.

" + } + ], + "documentation":"

Delete an existing to-do item.

" + } + }, + "shapes":{ + "CreateToDoInput":{ + "type":"structure", + "required":[ + "Title" + ], + "members":{ + "Title":{ + "shape":"String", + "documentation":"The title of the to-do item." + } + }, + "documentation":"

Container for to-do values.

" + }, + "String":{ + "type":"string" + }, + "ToDoClientException":{ + "type":"structure", + "members":{ + "message":{"shape":"ErrorMessage"} + }, + "exception":true, + "documentation":"

The API was called with invalid parameters. The error message will contain additional details about the cause.

" + }, + "ToDoInput":{ + "type":"structure", + "required":[ + "Id" + ], + "members":{ + "Id":{ + "shape":"String", + "location":"uri", + "locationName":"Id" + } + } + }, + "ToDoItem":{ + "type":"structure", + "members":{ + "Id":{ + "shape":"String", + "documentation":"Unique identifier" + }, + "Title":{ + "shape":"String", + "documentation":"The title of the to-do item." + }, + "Status":{ + "shape":"String", + "documentation":"The status of the to-do item. Either CREATING, READY, or DONE." + } + }, + "documentation":"A single to-do item." + }, + "ToDoList":{ + "type":"list", + "member":{ + "shape":"ToDoItem", + "documentation":"List of to-do items." + } + }, + "ToDoServerException":{ + "type":"structure", + "members":{ + "message":{"shape":"ErrorMessage"} + }, + "exception":true, + "documentation":"

A server-side error occurred during the API call. The error message will contain additional details about the cause.

" + }, + "ErrorMessage":{ + "type":"string" + } + } +} diff --git a/tests/unit/data/aws/todo/2015-04-01.waiters.json b/tests/unit/data/aws/todo/2015-04-01.waiters.json new file mode 100644 index 0000000000..9f90c97758 --- /dev/null +++ b/tests/unit/data/aws/todo/2015-04-01.waiters.json @@ -0,0 +1,24 @@ +{ + "version": 2, + "waiters": { + "ToDoReady": { + "delay": 20, + "operation": "GetToDo", + "maxAttempts": 25, + "acceptors": [ + { + "expected": "READY", + "matcher": "path", + "state": "success", + "argument": "ToDo.Status" + }, + { + "expected": "DONE", + "matcher": "path", + "state": "success", + "argument": "ToDo.Status" + } + ] + } + } +} diff --git a/tests/unit/data/resources/todo-2015-04-01.resources.json b/tests/unit/data/resources/todo-2015-04-01.resources.json new file mode 100644 index 0000000000..39a22ef2b8 --- /dev/null +++ b/tests/unit/data/resources/todo-2015-04-01.resources.json @@ -0,0 +1,72 @@ +{ + "service": { + "actions": { + "CreateToDo": { + "request": { "operation": "CreateToDo" }, + "resource": { + "type": "ToDo", + "identifiers": [ + { "target": "Id", "source": "response", "path": "Id" } + ] + } + } + }, + "has": { + "ToDo": { + "resource": { + "type": "ToDo", + "identifiers": [ + { "target": "Id", "source": "input" } + ] + } + } + }, + "hasMany": { + "ToDos": { + "request": { "operation": "DescribeToDos" }, + "resource": { + "type": "ToDo", + "identifiers": [ + { "target": "Id", "source": "response", "path": "ToDoList[].Id" } + ] + } + } + } + }, + "resources": { + "ToDo": { + "identifiers": [ + { "name": "Id" } + ], + "shape": "ToDoItem", + "actions": { + "Delete": { + "request": { + "operation": "DeleteToDo", + "params": [ + { "target": "Id", "source": "identifier", "name": "Id" } + ] + } + } + }, + "waiters": { + "Ready": { + "waiterName": "ToDoReady", + "params": [ + { "target": "Id", "source": "identifier", "name": "Id" } + ] + } + }, + "has": { + "MySelf": { + "resource": { + "type": "ToDo", + "identifiers": [ + { "target": "Id", "source": "data", "path": "Id" } + ] + } + } + } + } + } +} diff --git a/tests/unit/test_docs.py b/tests/unit/test_docs.py index def1a75862..6a84f88027 100644 --- a/tests/unit/test_docs.py +++ b/tests/unit/test_docs.py @@ -11,11 +11,17 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +import os + from botocore.compat import json, OrderedDict from botocore.model import DenormalizedStructureBuilder, ServiceModel +from botocore.session import get_session + +from boto3.docs import py_type_name, document_param_response, docs_for +from tests import mock, unittest, BaseTestCase + -from boto3.docs import py_type_name, document_structure -from tests import mock, BaseTestCase +ROOT = os.path.dirname(__file__) def load_json(data): @@ -82,6 +88,141 @@ def test_double(self): self.assertEqual('float', py_type_name('double')) +class TestDocumentService(unittest.TestCase): + """ + End-to-end tests of the documentation functionality that uses a dummy + service and resource JSON description. The tests check for specific + values in the output. + """ + def setUp(self): + self.session = get_session() + + loader = self.session.get_component('data_loader') + loader.data_path = os.path.join(ROOT, 'data') + + self.docs = docs_for( + 'todo', session=self.session, + resource_filename=os.path.join(ROOT, 'data', 'resources', + 'todo-2015-04-01.resources.json')) + + self.resource_docs = '' + if 'Service Resource' in self.docs: + self.resource_docs = self.docs.split('Service Resource')[-1] + + def assert_after(self, marker, expected, actual): + """ + Assert that the expected value occurs after a text marker in + the actual value. + """ + pos = actual.find(marker) + self.assertIn(expected, actual[pos:]) + + def test_removed_html(self): + self.assertNotIn('

', self.docs) + self.assertNotIn('