20th feb.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "06b10612",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The status code is 200\n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "\n",
    "url=\"https://crawler-test.com/\"\n",
    "\n",
    "response=requests.get(url)\n",
    "print('The status code is',response.status_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4138df40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<!DOCTYPE html>\n",
      "<html>\n",
      "  <head>\n",
      "    <title>Crawler Test Site</title>\n",
      "    \n",
      "      <meta content=\"en\" H\n"
     ]
    }
   ],
   "source": [
    "#to access first 100 char\n",
    "print(response.text[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "3bd9bdb6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Crawler Test Site\n"
     ]
    }
   ],
   "source": [
    "#to get the title\n",
    "#to access the site\n",
    "soup=BeautifulSoup(response.text,'html.parser')\n",
    "print(soup.find('title').text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8453c549",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Crawler Test Site\n"
     ]
    }
   ],
   "source": [
    "heading=soup.find('h1')\n",
    "print(heading.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "9d36fc4a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"/\" id=\"logo\">Crawler Test <span class=\"neon-effect\">two point oh!</span></a>\n"
     ]
    }
   ],
   "source": [
    "#to find the tag\n",
    "links=soup.find('a')\n",
    "print(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "97c74d6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'bs4.element.ResultSet'>\n",
      "----------------------------\n",
      "<a href=\"/\" id=\"logo\">Crawler Test <span class=\"neon-effect\">two point oh!</span></a>\n",
      "\n",
      "<a href=\"/mobile/separate_desktop\">Separate Desktop page with separate mobile and/or AMP</a>\n",
      "\n",
      "<a href=\"/mobile/desktop_with_AMP_as_mobile\">Separate Desktop page with AMP page as AMP and Mobile</a>\n",
      "\n",
      "<a href=\"/mobile/separate_desktop_with_different_h1\">Separate Desktop with different H1</a>\n",
      "\n",
      "<a href=\"/mobile/separate_desktop_with_different_title\">Separate Desktop with different title</a>\n",
      "\n",
      "<a href=\"/mobile/separate_desktop_with_different_wordcount\">Separate Desktop with different wordcount</a>\n",
      "\n",
      "<a href=\"/mobile/separate_desktop_with_different_links_in\">Separate Desktop with different links in</a>\n",
      "\n",
      "<a href=\"/mobile/separate_desktop_with_different_links_out\">Separate Desktop with different links out</a>\n",
      "\n",
      "<a href=\"/mobile/separate_desktop_with_mobile_not_subdomain\">Separate Desktop which links to a mobile page not on the mobile subdomain</a>\n",
      "\n",
      "<a href=\"/mobile/desktop_with_self_canonical_mobile_and_amp\">Separate Desktop with mobile and AMP which self canonicalises</a>\n",
      "\n",
      "<a href=\"/mobile/separate_mobile_with_mobile_not_subdomain\">Separate Mobile page not on the mobile subdomain</a>\n",
      "\n",
      "<a href=\"/mobile/dynamic\">A dynamically served page</a>\n",
      "\n",
      "<a href=\"/mobile/responsive\">A responsive page</a>\n",
      "\n",
      "<a href=\"/mobile/no_mobile_configuration\">No mobile configuration</a>\n",
      "\n",
      "<a href=\"/mobile/other_desktop_that_links_to_the_same_mobile_pages\">Other desktop page linking to the same mobile pages as other desktop</a>\n",
      "\n",
      "<a href=\"/mobile/amp_with_separate_mobile\">An AMP page which also has a dedicated mobile page</a>\n",
      "\n",
      "<a href=\"/mobile/responsive_with_amp\">Responsive with AMP</a>\n",
      "\n",
      "<a href=\"/mobile/amp_with_responsive\">AMP for responsive Page</a>\n",
      "\n",
      "<a href=\"/mobile/no_mobile_with_amp\">No mobile configuration with AMP</a>\n",
      "\n",
      "<a href=\"/mobile/amp_with_no_mobile\">AMP for no mobile configuration</a>\n",
      "\n",
      "<a href=\"/mobile/amp_no_references\">An AMP page with no canonical (or self-ref, and no inbound AMP links</a>\n",
      "\n",
      "<a href=\"/mobile/amp_as_desktop_amp_and_mobile\">AMP page as mobile self-referential</a>\n",
      "\n",
      "<a href=\"/mobile/separate_amp_with_self_canonical\">AMP page which self canonicalises</a>\n",
      "\n",
      "<a href=\"/mobile/separate_desktop_irregular_media\">Separate Desktop with Meta Irregular Media Pixel Size</a>\n",
      "\n",
      "<a href=\"/mobile/separate_desktop_response_header_alt\">Separate Desktop with Response Header Alt</a>\n",
      "\n",
      "<a href=\"/description_tags/description_with_whitespace\">Description Tag With Whitespace</a>\n",
      "\n",
      "<a href=\"/description_tags/missing_description\">Description Tag Missing</a>\n",
      "\n",
      "<a href=\"/description_tags/no_description_nosnippet\">Description Tag Missing With Meta Nosnippet</a>\n",
      "\n",
      "<a href=\"/description_tags/duplicate_description\">Description Tag Duplicate</a>\n",
      "\n",
      "<a href=\"/description_tags/duplicate_description/foo\">Description Tag Duplicate</a>\n",
      "\n",
      "<a href=\"/description_tags/duplicate_description_and_noindex\">Noindex and Description Tag Duplicate</a>\n",
      "\n",
      "<a href=\"/description_tags/duplicate_description_and_noindex/foo\">Noindex and Description Tag Duplicate</a>\n",
      "\n",
      "<a href=\"/description_tags/description_over_max\">Description Tag Too Long</a>\n",
      "\n",
      "<a href=\"/description_tags/short_meta_description\">Short Meta Description</a>\n",
      "\n",
      "<a href=\"/description_tags/description_http_equiv\">HTTP-equiv description</a>\n",
      "\n",
      "<a href=\"/encoding/page_titles_character_encoded\">Page Titles Character Encoded</a>\n",
      "\n",
      "<a href=\"/encoding/url_with_foreign_characters/בלהבלה\">URL with Foreign Characters - Hebrew</a>\n",
      "\n",
      "<a href=\"/encoding/url_with_foreign_characters/すべての単語が高校程度の辞書に載っている\">URL with Foreign Characters - Japanese</a>\n",
      "\n",
      "<a href=\"/encoding/url_with_foreign_characters/pchnąć-w-tę-łódź-jeża-lub-ośm-skrzyń-fig\">URL with Foreign Characters - Polish</a>\n",
      "\n",
      "<a href=\"/encoding/url_with_foreign_characters/Шеф-взъярён-тчк-щипцы-с-эхом-гудбай-Жюль\">URL with Foreign Characters - Cyrilic</a>\n",
      "\n",
      "<a href=\"/encoding/url_with_foreign_characters/Zwölf-große-Boxkämpfer-jagen-Viktor-quer-über-den-Sylter-Deich\">URL with Foreign Characters - German</a>\n",
      "\n",
      "<a href=\"/encoding/url_with_foreign_characters/Fabio-me-exige-sin-tapujos-que-añada-cerveza-al-whisky\">URL with Foreign Characters - Spanish</a>\n",
      "\n",
      "<a href=\"/encoding/url_with_foreign_characters/﴿محمد-رسول-الله-والذين-معه-أشداء\">URL with Foreign Characters - Arab</a>\n",
      "\n",
      "<a href=\"/encoding/url_with_foreign_characters/γράμματα-του-ισπανικού-αλφαβήτου-καθώς\">URL with Foreign Characters - Greek</a>\n",
      "\n",
      "<a href=\"/encoding/url_with_foreign_characters/asød-æada-ådjghf-gägfd-asödsads\">URL with Foreign Characters - Nordic</a>\n",
      "\n",
      "<a href=\"/encoding/double_encoded_url/Zw%25C3%25B6lf-gro%25C3%259Fe-Boxk%25C3%25A4mpfer-jagen-Viktor-quer-%25C3%25BCber-den-Sylter-Deich\">Double Encoded URL - German</a>\n",
      "\n",
      "<a href=\"/encoding/inconsistent_character_encoding\">Inconsistent Character Encoding</a>\n",
      "\n",
      "<a href=\"http://www.søkbar.no\" rel=\"nofollow\" target=\"_blank\">Foreign Character Domain</a>\n",
      "\n",
      "<a href=\"/encoding/url/encoded_hashbang%23abc\">Encoded hashbang</a>\n",
      "\n",
      "<a href=\"/titles/title_with_whitespace\">Title With Whitespace</a>\n",
      "\n",
      "<a href=\"/titles/empty_title\">Title Empty</a>\n",
      "\n",
      "<a href=\"/titles/missing_title\">Title Missing</a>\n",
      "\n",
      "<a href=\"/titles/duplicate_title\">Title Duplicates</a>\n",
      "\n",
      "<a href=\"/titles/duplicate_title/foo\">Title Duplicates</a>\n",
      "\n",
      "<a href=\"/titles/duplicate_title/foo\">Title Duplicates</a>\n",
      "\n",
      "<a href=\"/titles/duplicate_title/bar\">Title Duplicates</a>\n",
      "\n",
      "<a href=\"/titles/duplicate_title/baz\">Title Duplicates</a>\n",
      "\n",
      "<a href=\"/titles/duplicate_title_and_noindex/bat\">Noindex and Title Duplicates</a>\n",
      "\n",
      "<a href=\"/titles/duplicate_title_and_noindex/bak\">Noindex and Title Duplicates</a>\n",
      "\n",
      "<a href=\"/titles/title_over_max\">Title Too Long</a>\n",
      "\n",
      "<a href=\"/titles/title_warning\">Title Warning</a>\n",
      "\n",
      "<a href=\"/titles/page_title_length_n\">Page Title Length</a>\n",
      "\n",
      "<a href=\"/titles/page_title_width_n\">Page Title Width</a>\n",
      "\n",
      "<a href=\"/titles/page_title_leading_trailing_spaces\">Page Title Leading/Trailing Spaces</a>\n",
      "\n",
      "<a href=\"/titles/double_triple_quadruple_spaces\">Double  Triple   Quadruple    Spaces</a>\n",
      "\n",
      "<a href=\"/titles/svg_title\">SVG Title</a>\n",
      "\n",
      "<a href=\"/titles/forced_double_triple_quadruple_spaces\">\n",
      "        Forced Double  Triple   Quadruple    Spaces\n",
      "      </a>\n",
      "\n",
      "<a href=\"/robots_protocol/robots_excluded\">Robots.txt Disallowed</a>\n",
      "\n",
      "<a href=\"/robots_protocol/deepcrawl_excluded\">DeepCrawl Excluded</a>\n",
      "\n",
      "<a href=\"/robots_protocol/robots_excluded_duplicate_description\">Robots.txt Disallowed With Duplicate Description</a>\n",
      "\n",
      "<a href=\"/robots_protocol/robots_excluded_meta_noindex\">Robots.txt Disallowed With Meta Noindex</a>\n",
      "\n",
      "<a href=\"/robots_protocol/deepcrawl_ua_disallow/foo\">Robots.txt Disallowed For Deepcrawl User Agent</a>\n",
      "\n",
      "<a href=\"/robots_protocol/user_excluded\">User Excluded</a>\n",
      "\n",
      "<a href=\"/robots_protocol/meta_nofollow\">Meta Nofollow</a>\n",
      "\n",
      "<a href=\"/robots_protocol/meta_noarchive\">Meta Noarchive</a>\n",
      "\n",
      "<a href=\"/robots_protocol/meta_noindex\">Meta Noindex</a>\n",
      "\n",
      "<a href=\"/robots_protocol/meta_noindex_uppercase\">Meta Noindex Uppercase</a>\n",
      "\n",
      "<a href=\"/robots_protocol/x_robots_tag_noindex\">X-robots Noindex</a>\n",
      "\n",
      "<a href=\"/robots_protocol/page_allowed_with_robots\">Page allowed with robots.txt</a>\n",
      "\n",
      "<a href=\"/robots_protocol/robots_noindexed\">Robots Noindexed</a>\n",
      "\n",
      "<a href=\"/robots_protocol/robots_noindex_conflict\">Robots Noindex Conflict</a>\n",
      "\n",
      "<a href=\"/robots_protocol/robots_excluded_blank_line\">Robots.txt Disallowed Blank Line</a>\n",
      "\n",
      "<a href=\"/robots_protocol/robots_noindexed_and_robots_disallowed\">Robots Noindexed and Robots Disallowed</a>\n",
      "\n",
      "<a href=\"/robots_protocol/allowed_same_length\">Allowed Same Length</a>\n",
      "\n",
      "<a href=\"/robots_protocol/allowed_shorter\">Allowed Shorter</a>\n",
      "\n",
      "<a href=\"/robots_protocol/allowed_longer\">Allowed Longer</a>\n",
      "\n",
      "<a href=\"/robots_protocol/meta_robots_and_x_robots_conflict\">Meta robots and X-Robots conflict</a>\n",
      "\n",
      "<a href=\"/robots_protocol/robots_meta_none\">Robots Meta None</a>\n",
      "\n",
      "<a href=\"/robots_protocol/robots_meta_noodp_noydir_none_noindex\">Robots Meta Noodp + Noydir + None + Noindex</a>\n",
      "\n",
      "<a href=\"/robots_protocol/robots_meta_multiple_tags_noindex_nofollow\">Robots Meta Multiple Tags: noindex + nofollow</a>\n",
      "\n",
      "<a href=\"/robots_protocol/x_robots_multiple_directives\">X-Robots Multiple Directives</a>\n",
      "\n",
      "<a href=\"/robots_protocol/multiple_robots_directive_meta_tag\">Multiple Robots directives meta tag</a>\n",
      "\n",
      "<a href=\"/robots_protocol/multiple_googlebot_directive_meta_tag\">Multiple Googlebot directives meta tag</a>\n",
      "\n",
      "<a href=\"/robots_protocol/non_200_with_noindex\">Non-200 with Noindex</a>\n",
      "\n",
      "<a href=\"/robots_protocol/canonicalised_with_noindex\">Canonicalised with Noindex</a>\n",
      "\n",
      "<a href=\"/robots_protocol/canonicalised_with_non_200\">Canonicalised with Non-200</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_1\">Redirect 301</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_2\">Redirect Double 301</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_3_302\">Redirect 302</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_4_307\">Redirect 307</a>\n",
      "\n",
      "<a href=\"/redirects/disallowed_redirect\">Disallowed Redirect</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_chain_allowed\">Redirect Chain Allowed</a>\n",
      "\n",
      "<a href=\"/redirects/disallowed_redirect_target_redirect\">Disallowed Redirect Target</a>\n",
      "\n",
      "<a href=\"/redirects/infinite_redirect\">Infinite Redirect</a>\n",
      "\n",
      "<a href=\"/redirects/two_step_redirect_loop_1\">Infinite Redirect (2 step</a>\n",
      "\n",
      "<a href=\"/redirects/external_redirect\">External Redirect</a>\n",
      "\n",
      "<a href=\"/redirects/invalid_redirect\">Redirect 303 To 404</a>\n",
      "\n",
      "<a href=\"/redirects/meta_redirect_1\">Meta Redirect</a>\n",
      "\n",
      "<a href=\"/redirects/meta_redirect_2\">Meta Redirect - Example 2</a>\n",
      "\n",
      "<a href=\"/redirects/meta_redirect_3\">Meta Redirect - Example 3</a>\n",
      "\n",
      "<a href=\"/redirects/infinite_meta_redirect\">Infinite Meta Redirect Loop</a>\n",
      "\n",
      "<a href=\"/redirects/external_meta_redirect\">External Meta Redirect</a>\n",
      "\n",
      "<a href=\"/redirects/invalid_meta_redirect\">Invalid Meta Redirect</a>\n",
      "\n",
      "<a href=\"/redirects/header_refresh_redirect\">Header Refresh Redirect</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_to_404\">Redirect To 404 Http Status</a>\n",
      "\n",
      "<a href=\"/redirects/url_redirect_chains\">URL Redirect Chains</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_content\">Redirect Content</a>\n",
      "\n",
      "<a href=\"/redirects/external_redirect_chain1\">External Redirect Chain 1</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_300\">Redirect 300</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_303\">Redirect 303</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_304\">Redirect 304</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_305\">Redirect 305</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_306\">Redirect 306</a>\n",
      "\n",
      "<a href=\"/redirects/redirect_308\">Redirect 308</a>\n",
      "\n",
      "<a href=\"/javascript/window-location-internal\">javascript:window.location internal</a>\n",
      "\n",
      "<a href=\"/javascript/window-location-external\">javascript:window.location external</a>\n",
      "\n",
      "<a href=\"/javascript/window-location-function-absolute\">javascript:window.location function absolute</a>\n",
      "\n",
      "<a href=\"/javascript/window-location-function-relative\">javascript:window.location function relative</a>\n",
      "\n",
      "<a href=\"/links/broken_links_internal\">Broken Links Internal</a>\n",
      "\n",
      "<a href=\"/links/broken_links_external\">Broken Links External</a>\n",
      "\n",
      "<a href=\"/links/max_external_links\">Max External Links</a>\n",
      "\n",
      "<a href=\"/links/page_with_external_links\">Page with External Links</a>\n",
      "\n",
      "<a href=\"/links/nofollowed_page\" rel=\"nofollow\">Nofollowed Page</a>\n",
      "\n",
      "<a href=\"/links/nofollow_link_with_nofollowed_backlinks\" rel=\"nofollow\">Nofollow Link With Nofollowed Backlinks</a>\n",
      "\n",
      "<a href=\"/links/relative_link/a/b\">Relative Link</a>\n",
      "\n",
      "<a href=\"/links/relative_link_with_base/a/b\">Relative Link With Base</a>\n",
      "\n",
      "<a href=\"/links/image_links\">Image Links</a>\n",
      "\n",
      "<a href=\"/links/non_default_language\">Non Default Language</a>\n",
      "\n",
      "<a href=\"/links/meta_refresh\">Meta Refresh</a>\n",
      "\n",
      "<a href=\"/links/header_refresh\">Header Refresh</a>\n",
      "\n",
      "<a href=\"/links/external_links_to_disallwed_urls\">External Links to Disallowed URLs</a>\n",
      "\n",
      "<a href=\"/links/non_standard_links\">Non Standard Links</a>\n",
      "\n",
      "<a href=\"/links/repeated_external_links\">Repeated external links</a>\n",
      "\n",
      "<a href=\"/links/repeated_internal_links\">Repeated internal links</a>\n",
      "\n",
      "<a href=\"/links/links_with_quote_variations\">Links with Quote Variations</a>\n",
      "\n",
      "<a href=\"/links/whitespace_in_links\">Whitespace in Links</a>\n",
      "\n",
      "<a href=\"/links/comma_separated_attributes\" rel=\"nofollow,noindex\">Comma Separated Attributes</a>\n",
      "\n",
      "<a href=\"/links/nofollow_and_follow_link\">Nofollow &amp; Followed Link</a>\n",
      "\n",
      "<a href=\"//crawler-test.com/links/relative_protocol_page\">Relative Protocol Page</a>\n",
      "\n",
      "<a href=\"/javascript/window-location-onchange\">javascript:window.location onchange</a>\n",
      "\n",
      "<a href=\"/javascript/window-open\">javascript:window.open</a>\n",
      "\n",
      "<a href=\"/javascript/onmousedown\">javascript:onmousedown</a>\n",
      "\n",
      "<a href=\"/javascript/concatenatedlink\">javascript:concatenated link</a>\n",
      "\n",
      "<a href=\"/javascript/data-hreflink\">javascript:data-href link</a>\n",
      "\n",
      "<a href=\"/javascript/push_state\">javascript:push-state</a>\n",
      "\n",
      "<a href=\"/urls/double_slash//one\">Double Slash</a>\n",
      "\n",
      "<a href=\"/urls/double_slash////two\">Double Slash 2</a>\n",
      "\n",
      "<a href=\"/urls/double_slash//////three\">Double Slash 3</a>\n",
      "\n",
      "<a href=\"/urls/double_slash//////four//\">Double Slash 4</a>\n",
      "\n",
      "<a href=\"/urls/parameter_1_1?parameter_1=foo\">Parameter_1_1</a>\n",
      "\n",
      "<a href=\"/urls/parameter_1_2?parameter_x=x¶meter_1=foo\">Parameter_1_2</a>\n",
      "\n",
      "<a href=\"/urls/parameter_1_3?parameter_x=x¶meter_1=foo¶meter_y=y\">Parameter_1_3</a>\n",
      "\n",
      "<a href=\"/urls/parameter_2_1?parameter_1=foo\">Parameter_2_1</a>\n",
      "\n",
      "<a href=\"/urls/parameter_2_2?parameter_x=x¶meter_1=foo\">Parameter_2_2</a>\n",
      "\n",
      "<a href=\"/urls/parameter_2_3?parameter_x=x¶meter_1=foo¶meter_y=y\">Parameter_2_3</a>\n",
      "\n",
      "<a href=\"/urls/parameter?parameter_1=foo¶meter_1=bar\">Parameter Same Name Diff Value</a>\n",
      "\n",
      "<a href=\"/urls/parameter?parameter_1=foo¶meter_1=foo\">Parameter Same Name Same Value</a>\n",
      "\n",
      "<a href=\"/urls/url_with_spaces/URL with spaces\">Url With Spaces</a>\n",
      "\n",
      "<a href=\"/urls/url_with_trailing_space/%20\">Url With Trailing Space</a>\n",
      "\n",
      "<a href=\"/urls/url_with_encoded_trailing_space/ \">Url With Encoded Trailing Space</a>\n",
      "\n",
      "<a href=\"/urls/duplication_types/\">Url With Trailing Slash</a>\n",
      "\n",
      "<a href=\"/urls/duplication_types\">Url Without Trailing Slash</a>\n",
      "\n",
      "<a href=\"/urls/duplication_types?tracking=yes\">Url With Tracking Parameter</a>\n",
      "\n",
      "<a href=\"/urls/duplication_types/index.html\">Url Ending With /index.htm</a>\n",
      "\n",
      "<a href=\"/urls/duplication_types/duplication_types/\">Url With Duplicate Path</a>\n",
      "\n",
      "<a href=\"/urls/DUPLICATION_TYPES\">Url With Alternative Case</a>\n",
      "\n",
      "<a href=\"/urls/links_to_malformed_urls\">Links to Malformed URLs</a>\n",
      "\n",
      "<a href=\"/urls/paginated_page\">Paginated Pages</a>\n",
      "\n",
      "<a href=\"/urls/unlinked_paginated_page\">Unlinked Paginated Pages</a>\n",
      "\n",
      "<a href=\"/urls/paginated_and_noindex_page\">Paginated and Noindex Pages</a>\n",
      "\n",
      "<a href=\"/urls/links_to_non_html_filetypes\">Links to Non-HTML Filetypes</a>\n",
      "\n",
      "<a href=\"/urls/pages_with_hreflang\">Pages with HREFLANG</a>\n",
      "\n",
      "<a href=\"/urls/page_with_hreflang_header_ok\">Page with HREFLANG Header OK</a>\n",
      "\n",
      "<a href=\"/urls/page_with_hreflang_header_not_ok\">Page with HREFLANG Header Not OK</a>\n",
      "\n",
      "<a href=\"/urls/duplicate_page\">Duplicate Page</a>\n",
      "\n",
      "<a href=\"/urls/duplicate_page/foo\">Duplicate Page</a>\n",
      "\n",
      "<a href=\"/urls/duplicate_page/bar\">Duplicate Page</a>\n",
      "\n",
      "<a href=\"/urls/duplicate_page/baz\">Duplicate Page</a>\n",
      "\n",
      "<a href=\"/urls/with_session_id?sessionID=txTWJng82TKywFOU2Cjp\">Url With Session Id</a>\n",
      "\n",
      "<a href=\"/urls/page_url_length_n\">Page URL Length</a>\n",
      "\n",
      "<a href=\"/urls/url_with_fragment#fragment\">URL With Fragment</a>\n",
      "\n",
      "<a href=\"/url/URL%2c_with_encoded_reserved_character\">URL with encoded reserved character</a>\n",
      "\n",
      "<a href=\"/url/URL_with_encoded_unreserved_%63haracter\">URL with encoded un-reserved character</a>\n",
      "\n",
      "<a href=\"/url/with_encoded_space \">URL with encoded space</a>\n",
      "\n",
      "<a href=\"/url/with_encoded_ó_character\">URL with encoded ó character</a>\n",
      "\n",
      "<a href=\"/urls/directory_index/\">Directory index - blank</a>\n",
      "\n",
      "<a href=\"/urls/directory_index/index.htm\">Directory index - index.htm</a>\n",
      "\n",
      "<a href=\"/urls/directory_index/default.htm\">Directory index - default.htm</a>\n",
      "\n",
      "<a href=\"/urls/directory_index/index.html\">Directory index - index.html</a>\n",
      "\n",
      "<a href=\"/urls/directory_index/index\">Directory index - index</a>\n",
      "\n",
      "<a href=\"/infinite/\">Infinite URLs</a>\n",
      "\n",
      "<a href=\"/relative_base_source\">Relative Base</a>\n",
      "\n",
      "<a href=\"/one/two/three/four\">Four Paths</a>\n",
      "\n",
      "<a href=\"/one/two/three/four/five\">Five Paths</a>\n",
      "\n",
      "<a href=\"/one/two/three/four/five/six\">Six Paths</a>\n",
      "\n",
      "<a href=\"/one/two/three/four/five/six/seven\">Seven Paths</a>\n",
      "\n",
      "<a href=\"/path/1/path/2\">Double Path</a>\n",
      "\n",
      "<a href=\"/path/1/path/2/path/3\">Triple Path</a>\n",
      "\n",
      "<a href=\"/urls/multiple_slashes///200_404\">Multiple Slashes - 200 to 404</a>\n",
      "\n",
      "<a href=\"/urls/multiple_slashes///404_200\">Multiple Slashes - 404 to 200</a>\n",
      "\n",
      "<a href=\"https://crawler-test.com//urls/double_slash/disallowed_start\">Double Slash Disallowed Start</a>\n",
      "\n",
      "<a href=\"/urls/double_slash//disallowed_middle\">Double Slash Disallowed Middle</a>\n",
      "\n",
      "<a href=\"/urls/double_slash/disallowed_end//\">Double Slash Disallowed End</a>\n",
      "\n",
      "<a href=\"?parameter-on-hostname-root=parameter-value\">Parameter on Hostname Root</a>\n",
      "\n",
      "<a href=\"?removed_parameter=1&amp;retained_parameter=1\">Parameter Removal</a>\n",
      "\n",
      "<a href=\"/url/url_with:colon\">URL With Colon</a>\n",
      "\n",
      "<a href=\"relativeurl:withcolon\" target=\"_blank\">Relative URL With Colon</a>\n",
      "\n",
      "<a href=\"/canonical_tags/relative_root_canonical_tag\">Canonical Tag With Relative Root</a>\n",
      "\n",
      "<a href=\"/canonical_tags/relative_canonical_tag\">Canonical Tag With Relative Url</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag\">Canonical Tag</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag_uppercase\">Canonical Tag Uppercase</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/2\">Canonical Tag 2</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/3\">Canonical Tag 3</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/4\">Canonical Tag 4</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/5\">Canonical Tag 5</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/6\">Canonical Tag 6</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/7\">Canonical Tag 7</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/8\">Canonical Tag 8</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/9\">Canonical Tag 9</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/10\">Canonical Tag 10</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/11\">Canonical Tag 11</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/12\">Canonical Tag 12</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/13\">Canonical Tag 13</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/14\">Canonical Tag 14</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/15\">Canonical Tag 15</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/16\">Canonical Tag 16</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/17\">Canonical Tag 17</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/18\">Canonical Tag 18</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/19\">Canonical Tag 19</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/20\">Canonical Tag 20</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/21\">Canonical Tag 21</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/22\">Canonical Tag 22</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/23\">Canonical Tag 23</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/24\">Canonical Tag 24</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag/25\">Canonical Tag 25</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag_like_page\">Canonical Tag Like Page</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_duplicate_description\">Canonical And Duplicate Description</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag_in_header\">Canonical Tag In Header</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag_outside_head\">Canonical Tag Outside Head</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag_html_header_conflict\">Canonical Tag HTML/Header Conflict</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag_html_conflict\">Canonical Tag HTML Conflict</a>\n",
      "\n",
      "<a href=\"/canonical_tags/page_with_external_canonical\">Page With External Canonical</a>\n",
      "\n",
      "<a href=\"/canonical_tags/page_without_canonical_tag\">Page Without Canonical Tag</a>\n",
      "\n",
      "<a href=\"/canonical_tags/unlinked_canonical\">Unlinked_canonical</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag_og_url_conflict\">Canonical Tag/OG:URL Conflict</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_tag_with_self_reference\">Canonical Tag with Self Reference</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonicalised_to_disallowed_url\">Canonicalised to Disallowed URL</a>\n",
      "\n",
      "<a href=\"/canonical_tags/unlinked_canonical_header\">Unlinked Canonical URL in Header</a>\n",
      "\n",
      "<a href=\"/canonical_tags/non_head_canonical\">Non-Head Canonical</a>\n",
      "\n",
      "<a href=\"/canonical_tags/non_head_canonical_link\">Non-Head Canonical Link</a>\n",
      "\n",
      "<a href=\"/canonical_tags/non_head_canonical_link_2\">Non-Head Canonical Link 2</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_port_80\">Port :80 = http with no port</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_port_443\">Port :443 = http with no port</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_port_8080\">Port :8080 = http with no port</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_url_encoded_vs_non_encoded/café\">URL-encoded v non-encoded not canonicalised</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_url_encoded_vs_non_encoded/🍺/ist\">URL-encoded v non-encoded not canonicalised icon</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_prameter_key_is_case_sensitive?key=value\">Parameter key is case sensitive</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_prameter_value_is_case_sensitive?key=value\">Parameter value is case sensitive</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_url_fragments#url-fragment\">URL Fragments are not canonicalised</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_different_parameter_order?key2=value2&amp;key=value\">Different parameter orders are canonicalised</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_hostname_case_insensitive\">Hostname is case insensitive</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_protocol_case_insensitive\">Protocol is case insensitive</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_path_is_case_sensitive\">Path is case sensitive</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_url_with_slash\">URI = / is the same as naked domain</a>\n",
      "\n",
      "<a href=\"/canonical_tags/canonical_trailing_dot\">Trailing dot is the same as naked domain (extremely uncommon)</a>\n",
      "\n",
      "<a href=\"/status_codes/status_100\">100 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_101\">101 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_102\">102 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_200\">200 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_201\">201 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_202\">202 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_203\">203 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_204\">204 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_205\">205 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_206\">206 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_207\">207 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_226\">226 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_400\">400 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_401\">401 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_402\">402 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_403\">403 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_404\">404 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_405\">405 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_406\">406 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_407\">407 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_408\">408 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_409\">409 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_410\">410 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_411\">411 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_412\">412 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_413\">413 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_414\">414 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_415\">415 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_416\">416 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_417\">417 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_418\">418 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_419\">419 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_420\">420 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_421\">421 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_422\">422 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_423\">423 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_424\">424 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_426\">426 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_428\">428 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_429\">429 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_431\">431 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_440\">440 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_444\">444 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_449\">449 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_450\">450 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_451\">451 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_494\">494 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_495\">495 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_496\">496 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_497\">497 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_498\">498 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_499\">499 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_500\">500 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_501\">501 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_502\">502 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_503\">503 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_504\">504 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_505\">505 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_506\">506 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_507\">507 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_508\">508 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_509\">509 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_510\">510 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_511\">511 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_520\">520 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_598\">598 HTTP Status</a>\n",
      "\n",
      "<a href=\"/status_codes/status_599\">599 HTTP Status</a>\n",
      "\n",
      "<a href=\"/social_tags/open_graph_tags\">Open Graph Tags</a>\n",
      "\n",
      "<a href=\"/social_tags/twitter_card_page/1\">Twitter Card Page 1</a>\n",
      "\n",
      "<a href=\"/social_tags/twitter_card_page/2\">Twitter Card Page 2</a>\n",
      "\n",
      "<a href=\"/social_tags/og_no_twitter\">OG Description, Title, Image, No Twitter</a>\n",
      "\n",
      "<a href=\"/social_tags/max_twitter_card_description_length\">Max Twitter Card Description Length</a>\n",
      "\n",
      "<a href=\"/content/custom_text\">Custom Text</a>\n",
      "\n",
      "<a href=\"/content/error_page\">Error Page</a>\n",
      "\n",
      "<a href=\"/content/above_min_content_volume\">Above Min Content Volume</a>\n",
      "\n",
      "<a href=\"/content/no_h1\">No H1s</a>\n",
      "\n",
      "<a href=\"/content/h1_in_img\">H1 in img</a>\n",
      "\n",
      "<a href=\"/content/mult_h1\">Mult H1s</a>\n",
      "\n",
      "<a href=\"/content/page_html_size_n\">Page HTML Size (n KB)</a>\n",
      "\n",
      "<a href=\"/content/page_content_size_n\">Page Content Size (n KB)</a>\n",
      "\n",
      "<a href=\"/content/meta_content_type_text_html\">Meta content type - text/html</a>\n",
      "\n",
      "<a href=\"/content/meta_content_type_malformed\">Meta content type - malformed</a>\n",
      "\n",
      "<a href=\"/content/word_count_100_words\">Word Count - 100 Words</a>\n",
      "\n",
      "<a href=\"/content/word_count_number\">Word Count - Number</a>\n",
      "\n",
      "<a href=\"/content/word_count_hyphenated\">Word Count - Hyphenated</a>\n",
      "\n",
      "<a href=\"/content/word_count_symbols\">Word Count - Symbols</a>\n",
      "\n",
      "<a href=\"/content/word_count_script\">Word Count - Script</a>\n",
      "\n",
      "<a href=\"/content/custom_extraction_text\">Custom extraction text</a>\n",
      "\n",
      "<a href=\"/content/multiple_titles_and_descriptions\">Multiple titles and descriptions</a>\n",
      "\n",
      "<a href=\"/content/title_with_newline_quote_doublequote_and_comma_characters\">Title with newline, quote, doublequote and comma characters</a>\n",
      "\n",
      "<a href=\"/content/header_content_type_malformed\">Header Content Type Malformed</a>\n",
      "\n",
      "<a href=\"/javascript/window-location-onclick\">javascript:window.location onclick</a>\n",
      "\n",
      "<a href=\"/javascript/onclick-reveals-element-programmatically-added-onclick\">javascript:onclick reveals new element with programmatically added onclick event</a>\n",
      "\n",
      "<a href=\"/javascript/ajax-return-data\">javascript:make AJAX call and return data in page</a>\n",
      "\n",
      "<a href=\"/javascript/dynamically-inserted-text\">javascript:dynamically inserted text</a>\n",
      "\n",
      "<a href=\"/javascript/dynamically-inserted-text-meta-data\">javascript:dynamically inserted text meta data</a>\n",
      "\n",
      "<a href=\"/javascript/dynamically-inserted-nofollow\">javascript:dynamically inserted nofollow</a>\n",
      "\n",
      "<a href=\"/javascript/onload-added-title\">javascript:on load inserted title</a>\n",
      "\n",
      "<a href=\"/javascript/onload-inserted-canonical\">javascript:on load inserted canonical URL</a>\n",
      "\n",
      "<a href=\"/javascript/dialog_window\">javascript:dialog window</a>\n",
      "\n",
      "<a href=\"/javascript/alert_box\">javascript:alert box</a>\n",
      "\n",
      "<a href=\"/javascript/ad_script\">javascript:ad script</a>\n",
      "\n",
      "<a href=\"/javascript/analytics_script\">javascript:analytics script</a>\n",
      "\n",
      "<a href=\"/javascript/renderer_timeout/1\">javascript:Rendering test 1</a>\n",
      "\n",
      "<a href=\"/javascript/renderer_timeout/2\">javascript:Rendering test 2</a>\n",
      "\n",
      "<a href=\"/javascript/renderer_timeout/3\">javascript:Rendering test 3</a>\n",
      "\n",
      "<a href=\"/javascript/renderer_timeout/4\">javascript:Rendering test 4</a>\n",
      "\n",
      "<a href=\"/javascript/renderer_timeout/5\">javascript:Rendering test 5</a>\n",
      "\n",
      "<a href=\"/content/non_secure_form_fields_text\">Non-secure Form Fields - Text</a>\n",
      "\n",
      "<a href=\"/content/non_secure_form_fields_email\">Non-secure Form Fields - Email</a>\n",
      "\n",
      "<a href=\"/content/non_secure_form_fields_search\">Non-secure Form Fields - Search</a>\n",
      "\n",
      "<a href=\"/content/non_secure_form_fields_number\">Non-secure Form Fields - Number</a>\n",
      "\n",
      "<a href=\"/content/non_secure_form_fields_tel\">Non-secure Form Fields - Tel</a>\n",
      "\n",
      "<a href=\"/content/non_secure_form_fields_url\">Non-secure Form Fields - URL</a>\n",
      "\n",
      "<a href=\"/content/non_secure_form_fields_textarea\">Non-secure Form Fields - Textarea</a>\n",
      "\n",
      "<a href=\"/content/non_secure_form_fields_password_and_cc\">Non-secure Form Fields - Password and Credit Cards</a>\n",
      "\n",
      "<a href=\"/other/crawler_user_agent\">Crawler User Agent</a>\n",
      "\n",
      "<a href=\"/other/crawler_ip_address\">Crawler IP Address</a>\n",
      "\n",
      "<a href=\"/other/conflicting_language_tags\">Conflicting Language Tags</a>\n",
      "\n",
      "<a href=\"/other/page_load_time_n\">Page Load Time</a>\n",
      "\n",
      "<a href=\"/other/crawler_request_headers\">Crawler Request Headers</a>\n",
      "\n",
      "<a href=\"/other/expiring_page_for_removed_test/1708418684.4472957\">Expiring Page for Removed Test</a>\n",
      "\n",
      "<a href=\"/other/duplicated_body_content_1\">Duplicated Body Content 1</a>\n",
      "\n",
      "<a href=\"/other/duplicated_body_content_2\">Duplicated Body Content 2</a>\n",
      "\n",
      "<a href=\"/other/string_width/512/string\">String Width in Pixels</a>\n",
      "\n",
      "<a href=\"/other/script_tag_contents\">Script Tag Contents</a>\n",
      "\n",
      "<a href=\"/other/noodp_noydir_tags\">NoODP and NoYDir Tags</a>\n",
      "\n",
      "<a href=\"/other/page_with_hsts_headers\">Page with HSTS Headers</a>\n",
      "\n",
      "<a href=\"https://subdomain.crawler-test.com\">Subdomain</a>\n",
      "\n",
      "<a href=\"https://invalid.crawler-test.com\">Invalid Subdomain</a>\n",
      "\n",
      "<a href=\"http://crawler-test.com/\">Crawler Test HTTP/Non-WWW</a>\n",
      "\n",
      "<a href=\"http://crawler-test.com\">Crawler Test HTTP</a>\n",
      "\n",
      "<a href=\"https://crawler-test.com\">Crawler Test HTTPS</a>\n",
      "\n",
      "<a href=\"/other/in_web_linking\">Page linking from web</a>\n",
      "\n",
      "<a href=\"/other/in_web_linked\">Page linked from web</a>\n",
      "\n",
      "<a href=\"/other/typo_in_head\">Broken HTML - Typo inside head</a>\n",
      "\n",
      "<a href=\"/other/unfinished_tag_in_head\">Broken HTML - Unfinished tag in head</a>\n",
      "\n",
      "<a href=\"/other/non_head_tag_in_head\">Broken HTML - Non head tag in head</a>\n",
      "\n",
      "<a href=\"/other/link_tag_in_body\">Broken HTML - Link tag in body</a>\n",
      "\n",
      "<a href=\"/other/basic_auth\">Basic Auth</a>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#to print all links\n",
    "all_links=soup.find_all('a')\n",
    "print(type(all_links))\n",
    "print('----------------------------')\n",
    "for val in all_links:\n",
    "    print(val)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "9872d609",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div id=\"header\">\n",
      "<a href=\"/\" id=\"logo\">Crawler Test <span class=\"neon-effect\">two point oh!</span></a>\n",
      "<div style=\"position:absolute;right:520px;top:-4px;\"></div>\n",
      "</div>\n"
     ]
    }
   ],
   "source": [
    "#to find element by ID\n",
    "head=soup.find(id='header')\n",
    "print(head)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e5d93da9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"/\" id=\"logo\">Crawler Test <span class=\"neon-effect\">two point oh!</span></a>\n"
     ]
    }
   ],
   "source": [
    "#to access a tag in particular ID\n",
    "a=head.find('a')\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "d5781101",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div class=\"row side-collapsed\">\n",
      "<div class=\"large-4 columns\">\n",
      "<div class=\"panel\">\n",
      "<div class=\"panel-header\">\n",
      "<h3>Mobile</h3>\n",
      "</div>\n",
      "<a href=\"/mobile/separate_desktop\">Separate Desktop page with separate mobile and/or AMP</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/desktop_with_AMP_as_mobile\">Separate Desktop page with AMP page as AMP and Mobile</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/separate_desktop_with_different_h1\">Separate Desktop with different H1</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/separate_desktop_with_different_title\">Separate Desktop with different title</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/separate_desktop_with_different_wordcount\">Separate Desktop with different wordcount</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/separate_desktop_with_different_links_in\">Separate Desktop with different links in</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/separate_desktop_with_different_links_out\">Separate Desktop with different links out</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/separate_desktop_with_mobile_not_subdomain\">Separate Desktop which links to a mobile page not on the mobile subdomain</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/desktop_with_self_canonical_mobile_and_amp\">Separate Desktop with mobile and AMP which self canonicalises</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/separate_mobile_with_mobile_not_subdomain\">Separate Mobile page not on the mobile subdomain</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/dynamic\">A dynamically served page</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/responsive\">A responsive page</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/no_mobile_configuration\">No mobile configuration</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/other_desktop_that_links_to_the_same_mobile_pages\">Other desktop page linking to the same mobile pages as other desktop</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/amp_with_separate_mobile\">An AMP page which also has a dedicated mobile page</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/responsive_with_amp\">Responsive with AMP</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/amp_with_responsive\">AMP for responsive Page</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/no_mobile_with_amp\">No mobile configuration with AMP</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/amp_with_no_mobile\">AMP for no mobile configuration</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/amp_no_references\">An AMP page with no canonical (or self-ref, and no inbound AMP links</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/amp_as_desktop_amp_and_mobile\">AMP page as mobile self-referential</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/separate_amp_with_self_canonical\">AMP page which self canonicalises</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/separate_desktop_irregular_media\">Separate Desktop with Meta Irregular Media Pixel Size</a>\n",
      "<br/>\n",
      "<a href=\"/mobile/separate_desktop_response_header_alt\">Separate Desktop with Response Header Alt</a>\n",
      "<br/>\n",
      "</div>\n",
      "</div>\n",
      "<div class=\"large-4 columns\">\n",
      "<div class=\"panel\">\n",
      "<div class=\"panel-header\">\n",
      "<h3>Description Tags</h3>\n",
      "</div>\n",
      "<a href=\"/description_tags/description_with_whitespace\">Description Tag With Whitespace</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/missing_description\">Description Tag Missing</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/no_description_nosnippet\">Description Tag Missing With Meta Nosnippet</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/duplicate_description\">Description Tag Duplicate</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/duplicate_description/foo\">Description Tag Duplicate</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/duplicate_description_and_noindex\">Noindex and Description Tag Duplicate</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/duplicate_description_and_noindex/foo\">Noindex and Description Tag Duplicate</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/description_over_max\">Description Tag Too Long</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/short_meta_description\">Short Meta Description</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/description_http_equiv\">HTTP-equiv description</a>\n",
      "<br/>\n",
      "</div>\n",
      "<div class=\"panel\">\n",
      "<div class=\"panel-header\">\n",
      "<h3>Encoding</h3>\n",
      "</div>\n",
      "<a href=\"/encoding/page_titles_character_encoded\">Page Titles Character Encoded</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/url_with_foreign_characters/בלהבלה\">URL with Foreign Characters - Hebrew</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/url_with_foreign_characters/すべての単語が高校程度の辞書に載っている\">URL with Foreign Characters - Japanese</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/url_with_foreign_characters/pchnąć-w-tę-łódź-jeża-lub-ośm-skrzyń-fig\">URL with Foreign Characters - Polish</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/url_with_foreign_characters/Шеф-взъярён-тчк-щипцы-с-эхом-гудбай-Жюль\">URL with Foreign Characters - Cyrilic</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/url_with_foreign_characters/Zwölf-große-Boxkämpfer-jagen-Viktor-quer-über-den-Sylter-Deich\">URL with Foreign Characters - German</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/url_with_foreign_characters/Fabio-me-exige-sin-tapujos-que-añada-cerveza-al-whisky\">URL with Foreign Characters - Spanish</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/url_with_foreign_characters/﴿محمد-رسول-الله-والذين-معه-أشداء\">URL with Foreign Characters - Arab</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/url_with_foreign_characters/γράμματα-του-ισπανικού-αλφαβήτου-καθώς\">URL with Foreign Characters - Greek</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/url_with_foreign_characters/asød-æada-ådjghf-gägfd-asödsads\">URL with Foreign Characters - Nordic</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/double_encoded_url/Zw%25C3%25B6lf-gro%25C3%259Fe-Boxk%25C3%25A4mpfer-jagen-Viktor-quer-%25C3%25BCber-den-Sylter-Deich\">Double Encoded URL - German</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/inconsistent_character_encoding\">Inconsistent Character Encoding</a>\n",
      "<br/>\n",
      "<a href=\"http://www.søkbar.no\" rel=\"nofollow\" target=\"_blank\">Foreign Character Domain</a>\n",
      "<br/>\n",
      "<a href=\"/encoding/url/encoded_hashbang%23abc\">Encoded hashbang</a>\n",
      "<br/>\n",
      "</div>\n",
      "</div>\n",
      "<div class=\"large-4 columns\">\n",
      "<div class=\"panel\">\n",
      "<div class=\"panel-header\">\n",
      "<h3>Titles</h3>\n",
      "</div>\n",
      "<a href=\"/titles/title_with_whitespace\">Title With Whitespace</a>\n",
      "<br/>\n",
      "<a href=\"/titles/empty_title\">Title Empty</a>\n",
      "<br/>\n",
      "<a href=\"/titles/missing_title\">Title Missing</a>\n",
      "<br/>\n",
      "<a href=\"/titles/duplicate_title\">Title Duplicates</a>\n",
      "<br/>\n",
      "<a href=\"/titles/duplicate_title/foo\">Title Duplicates</a>\n",
      "<br/>\n",
      "<a href=\"/titles/duplicate_title/foo\">Title Duplicates</a>\n",
      "<br/>\n",
      "<a href=\"/titles/duplicate_title/bar\">Title Duplicates</a>\n",
      "<br/>\n",
      "<a href=\"/titles/duplicate_title/baz\">Title Duplicates</a>\n",
      "<br/>\n",
      "<a href=\"/titles/duplicate_title_and_noindex/bat\">Noindex and Title Duplicates</a>\n",
      "<br/>\n",
      "<a href=\"/titles/duplicate_title_and_noindex/bak\">Noindex and Title Duplicates</a>\n",
      "<br/>\n",
      "<a href=\"/titles/title_over_max\">Title Too Long</a>\n",
      "<br/>\n",
      "<a href=\"/titles/title_warning\">Title Warning</a>\n",
      "<br/>\n",
      "<a href=\"/titles/page_title_length_n\">Page Title Length</a>\n",
      "<br/>\n",
      "<a href=\"/titles/page_title_width_n\">Page Title Width</a>\n",
      "<br/>\n",
      "<a href=\"/titles/page_title_leading_trailing_spaces\">Page Title Leading/Trailing Spaces</a>\n",
      "<br/>\n",
      "<a href=\"/titles/double_triple_quadruple_spaces\">Double  Triple   Quadruple    Spaces</a>\n",
      "<br/>\n",
      "<a href=\"/titles/svg_title\">SVG Title</a>\n",
      "<br/>\n",
      "<a href=\"/titles/forced_double_triple_quadruple_spaces\">\n",
      "        Forced Double  Triple   Quadruple    Spaces\n",
      "      </a>\n",
      "</div>\n",
      "</div>\n",
      "</div>\n"
     ]
    }
   ],
   "source": [
    "# find the element based on class\n",
    "class_based=soup.find(class_='row side-collapsed')\n",
    "print(class_based)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "bdaee8c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Mobile\n",
      "\n"
     ]
    }
   ],
   "source": [
    "class_based=soup.find(class_='panel-header')\n",
    "print(class_based.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "c5b52710",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mobile\n",
      "Description Tags\n",
      "Encoding\n",
      "Titles\n",
      "Robots Protocol\n",
      "Redirects\n",
      "Links\n",
      "URLs\n",
      "Canonical Tags\n",
      "Status Codes\n",
      "Social Tags\n",
      "Content\n",
      "Other\n"
     ]
    }
   ],
   "source": [
    "headings =soup.find_all('div',{'class':'panel-header'})\n",
    "for val in headings:\n",
    "    h3 = val.find('h3')\n",
    "    print(h3.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "a981df57",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mobile\n",
      "Description Tags\n",
      "Encoding\n",
      "Titles\n",
      "Robots Protocol\n",
      "Redirects\n",
      "Links\n",
      "URLs\n",
      "Canonical Tags\n",
      "Status Codes\n",
      "Social Tags\n",
      "Content\n",
      "Other\n"
     ]
    }
   ],
   "source": [
    "s=soup.find_all('h3')\n",
    "for i in s:\n",
    "    print(i.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "3be24ba5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div class=\"panel\">\n",
      "<div class=\"panel-header\">\n",
      "<h3>Description Tags</h3>\n",
      "</div>\n",
      "<a href=\"/description_tags/description_with_whitespace\">Description Tag With Whitespace</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/missing_description\">Description Tag Missing</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/no_description_nosnippet\">Description Tag Missing With Meta Nosnippet</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/duplicate_description\">Description Tag Duplicate</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/duplicate_description/foo\">Description Tag Duplicate</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/duplicate_description_and_noindex\">Noindex and Description Tag Duplicate</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/duplicate_description_and_noindex/foo\">Noindex and Description Tag Duplicate</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/description_over_max\">Description Tag Too Long</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/short_meta_description\">Short Meta Description</a>\n",
      "<br/>\n",
      "<a href=\"/description_tags/description_http_equiv\">HTTP-equiv description</a>\n",
      "<br/>\n",
      "</div>\n"
     ]
    }
   ],
   "source": [
    "z=soup.find_all(class_='panel')\n",
    "print(z[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "189ae2aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Description Tags\n",
      "\n",
      "Description Tag With Whitespace\n",
      "\n",
      "Description Tag Missing\n",
      "\n",
      "Description Tag Missing With Meta Nosnippet\n",
      "\n",
      "Description Tag Duplicate\n",
      "\n",
      "Description Tag Duplicate\n",
      "\n",
      "Noindex and Description Tag Duplicate\n",
      "\n",
      "Noindex and Description Tag Duplicate\n",
      "\n",
      "Description Tag Too Long\n",
      "\n",
      "Short Meta Description\n",
      "\n",
      "HTTP-equiv description\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(z[1].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "5037f7e9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<h3>Mobile</h3>, <h3>Description Tags</h3>, <h3>Encoding</h3>, <h3>Titles</h3>, <h3>Robots Protocol</h3>, <h3>Redirects</h3>, <h3>Links</h3>, <h3>URLs</h3>, <h3>Canonical Tags</h3>, <h3>Status Codes</h3>, <h3>Social Tags</h3>, <h3>Content</h3>, <h3>Other</h3>]\n"
     ]
    }
   ],
   "source": [
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "a8fcb54c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<h3>Description Tags</h3>\n"
     ]
    }
   ],
   "source": [
    "print(s[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "4d6c1db2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"/description_tags/description_with_whitespace\">Description Tag With Whitespace</a>\n",
      "<a href=\"/description_tags/missing_description\">Description Tag Missing</a>\n",
      "<a href=\"/description_tags/no_description_nosnippet\">Description Tag Missing With Meta Nosnippet</a>\n",
      "<a href=\"/description_tags/duplicate_description\">Description Tag Duplicate</a>\n",
      "<a href=\"/description_tags/duplicate_description/foo\">Description Tag Duplicate</a>\n",
      "<a href=\"/description_tags/duplicate_description_and_noindex\">Noindex and Description Tag Duplicate</a>\n",
      "<a href=\"/description_tags/duplicate_description_and_noindex/foo\">Noindex and Description Tag Duplicate</a>\n",
      "<a href=\"/description_tags/description_over_max\">Description Tag Too Long</a>\n",
      "<a href=\"/description_tags/short_meta_description\">Short Meta Description</a>\n",
      "<a href=\"/description_tags/description_http_equiv\">HTTP-equiv description</a>\n"
     ]
    }
   ],
   "source": [
    "#p=soup.find_all(class_='panel')\n",
    "d=z[1]\n",
    "for i in d.find_all('a'):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "4100e551",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/description_tags/description_with_whitespace\n",
      "/description_tags/missing_description\n",
      "/description_tags/no_description_nosnippet\n",
      "/description_tags/duplicate_description\n",
      "/description_tags/duplicate_description/foo\n",
      "/description_tags/duplicate_description_and_noindex\n",
      "/description_tags/duplicate_description_and_noindex/foo\n",
      "/description_tags/description_over_max\n",
      "/description_tags/short_meta_description\n",
      "/description_tags/description_http_equiv\n"
     ]
    }
   ],
   "source": [
    "#to get links in href\n",
    "for i in d.find_all('a'):\n",
    "    print(i.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "eca95213",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://crawler-test.com//description_tags/description_with_whitespace\n",
      "https://crawler-test.com//description_tags/missing_description\n",
      "https://crawler-test.com//description_tags/no_description_nosnippet\n",
      "https://crawler-test.com//description_tags/duplicate_description\n",
      "https://crawler-test.com//description_tags/duplicate_description/foo\n",
      "https://crawler-test.com//description_tags/duplicate_description_and_noindex\n",
      "https://crawler-test.com//description_tags/duplicate_description_and_noindex/foo\n",
      "https://crawler-test.com//description_tags/description_over_max\n",
      "https://crawler-test.com//description_tags/short_meta_description\n",
      "https://crawler-test.com//description_tags/description_http_equiv\n"
     ]
    }
   ],
   "source": [
    "for i in d.find_all('a'):\n",
    "    print('https://crawler-test.com/'+str(i.get('href')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b34d3610",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}