rule_based_skill_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0a721886",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Credit to Edward Ross - https://skeptric.com/notebooks/Parsing%20Experience%20from%20Adzuna%20Job%20Ads.html\n",
    "import re\n",
    "import pandas as pd\n",
    "import spacy\n",
    "from spacy.util import filter_spans\n",
    "from spacy.tokens import Span\n",
    "from spacy.matcher import Matcher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b997bfd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy import displacy\n",
    "from IPython.display import HTML, display\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "18e2e3d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import en_core_web_sm\n",
    "nlp = en_core_web_sm.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "22401b3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('outputs/data_science_jobs.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0e0bf3ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(df.columns[0], axis = 1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "371e18da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Location</th>\n",
       "      <th>Job Board</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>852</th>\n",
       "      <td>HEAD OF ANALYTICS &amp; INSIGHTS/DATA SCIENTIST</td>\n",
       "      <td>Saje Natural Wellness</td>\n",
       "      <td>For over 25 years we’ve been helping our commu...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>Machine Learning Engineer</td>\n",
       "      <td>Aviva</td>\n",
       "      <td>About the job Individually we are people  but ...</td>\n",
       "      <td>Markham</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>627</th>\n",
       "      <td>Data Engineer / Power BI Developer, Omnia AI</td>\n",
       "      <td>Deloitte\\n</td>\n",
       "      <td>Job Type: Permanent Primary Location: Vancouve...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>Junior Business Intelligence Developer - Data ...</td>\n",
       "      <td>CAPRiser Group</td>\n",
       "      <td>Our subsidiary is in PropTech and real estate ...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>Data Scientist Technical Lead (Montreal), Inta...</td>\n",
       "      <td>Intact\\n</td>\n",
       "      <td>Who needs insurance? Everybody. That keeps us ...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>Bio informaticien/Bioinformatics scientist</td>\n",
       "      <td>GenAIz\\n</td>\n",
       "      <td>Avant le SRAS-CoV-2  un vaccin qui était dével...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Awakedata</td>\n",
       "      <td>RESPONSIBILITIES: Develop state-of-the-art com...</td>\n",
       "      <td>Burnaby</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>448</th>\n",
       "      <td>Scientifique de données / Data Scientist</td>\n",
       "      <td>BusPatrol\\n</td>\n",
       "      <td>Qui sommes-nous : BusPatrouille est une entrep...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>634</th>\n",
       "      <td>Data Management Specialist - Immunotoxicology</td>\n",
       "      <td>Charles River Laboratories\\n</td>\n",
       "      <td>For 70 years  Charles River employees have wor...</td>\n",
       "      <td>Senneville</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>748</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>F8th</td>\n",
       "      <td>We're transforming the Cyber Security industry...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>831</th>\n",
       "      <td>Senior Professional Environmental Scientist (G...</td>\n",
       "      <td>Madrone Environmental Services Ltd</td>\n",
       "      <td>Senior Professional Environmental Scientist an...</td>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Novisto</td>\n",
       "      <td>At Novisto  our vision is to be the world's le...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Peritus.ai</td>\n",
       "      <td>About Peritus.ai   Peritus is dedicated to bui...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Junior Project Scientist / Junior Project Tech...</td>\n",
       "      <td>SNC-Lavalin\\n</td>\n",
       "      <td>Junior Project Scientist / Junior Project Tech...</td>\n",
       "      <td>Fort St. John</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Data Engineer - 313378</td>\n",
       "      <td>Procom\\n</td>\n",
       "      <td>Ingénieur de données  Dans le cadre de ses ent...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>Senior Data Analyst</td>\n",
       "      <td>ZTR</td>\n",
       "      <td>Are you passionate about implementing and main...</td>\n",
       "      <td>London</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>Data Scientist - 05/12/21</td>\n",
       "      <td>Acerta Analytics Solutions Inc\\n</td>\n",
       "      <td>Acerta’s machine learning platforms leverage a...</td>\n",
       "      <td>Kitchener</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577</th>\n",
       "      <td>Senior Data Engineer - Remote</td>\n",
       "      <td>Koho\\n</td>\n",
       "      <td>About KOHO KOHO is a quickly scaling FinTech c...</td>\n",
       "      <td>Canada</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341</th>\n",
       "      <td>DATA SCIENTIST</td>\n",
       "      <td>Leger\\n</td>\n",
       "      <td>WHO WE ARE  At Leger  we know Canadians. With ...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>Data Science Manager, Omnia AI</td>\n",
       "      <td>Deloitte\\n</td>\n",
       "      <td>Job Type: Permanent Primary Location: Toronto ...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416</th>\n",
       "      <td>DATA ENGINEER</td>\n",
       "      <td>The Globe and Mail\\n</td>\n",
       "      <td>DATA ENGINEER POSITION CODE: 2021-063 LOCATION...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Mariner Partners\\n</td>\n",
       "      <td>Company Description As one of the largest prof...</td>\n",
       "      <td>Halifax</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>733</th>\n",
       "      <td>Principal Data Scientist</td>\n",
       "      <td>Veeva Systems</td>\n",
       "      <td>Veeva [NYSE: VEEV] is the leader in cloud-base...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>Data Engineer - Mississauga</td>\n",
       "      <td>Randstad\\n</td>\n",
       "      <td>Our client  located in Mississauga Ontario is ...</td>\n",
       "      <td>Mississauga</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>708</th>\n",
       "      <td>Product Development Confectionary Scientist</td>\n",
       "      <td>Cronos Group Inc.\\n</td>\n",
       "      <td>At Cronos Group  we hire talented people who t...</td>\n",
       "      <td>Stayner</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Machine Learning Scientist</td>\n",
       "      <td>Amazon Dev Centre Canada ULC\\n</td>\n",
       "      <td>Bachelor’s degree in Computer Science  Statist...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>Data Engineering Intern (Fall 2021) - Toronto</td>\n",
       "      <td>Coursera\\n</td>\n",
       "      <td>Are you ready to join a team of tech-savvy mus...</td>\n",
       "      <td>Remote</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>537</th>\n",
       "      <td>Manager, Data Science - Research &amp; Analytics T...</td>\n",
       "      <td>Tonal\\n</td>\n",
       "      <td>Who We Are  Tonal is the smartest home gym and...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>639</th>\n",
       "      <td>Ingénieur de données senior / Sr. Data Engineer</td>\n",
       "      <td>McKesson Canada Corporation / La Corporation M...</td>\n",
       "      <td>English version to follow  McKesson touche la ...</td>\n",
       "      <td>Saint-Laurent</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>669</th>\n",
       "      <td>Solution Architect (Big Data) - Canada</td>\n",
       "      <td>Wavicle Data Solutions\\n</td>\n",
       "      <td>Wavicle Data Solutions designs and delivers da...</td>\n",
       "      <td>Remote</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>Data Analyst Health Surveillance - BC Centre f...</td>\n",
       "      <td>PHSA\\n</td>\n",
       "      <td>Data Analyst Health Surveillance Epidemiology ...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>Optimization Specialist, Analytics</td>\n",
       "      <td>Torstar</td>\n",
       "      <td>Company Description  Torstar Corporation is a ...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>761</th>\n",
       "      <td>Machine Learning Engineer / Data Scientist</td>\n",
       "      <td>TerraSense Analytics Ltd</td>\n",
       "      <td>CATCHY INTRO  Honestly  we’ve been brainstormi...</td>\n",
       "      <td>Kelowna</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>Data Engineer, Data Lake</td>\n",
       "      <td>Altus Group\\n</td>\n",
       "      <td>Job Category:  Software  Opportunity Awaits at...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>686</th>\n",
       "      <td>Product Manager - Data Science - Toronto Hub</td>\n",
       "      <td>Veeva Systems</td>\n",
       "      <td>Veeva [NYSE: VEEV] is the leader in cloud-base...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Oxford Properties</td>\n",
       "      <td>Why join us?  Are you looking to join an innov...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>726</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Pinterest\\n</td>\n",
       "      <td>About Pinterest:  Millions of people across th...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>Cloud Data Engineer</td>\n",
       "      <td>Digital On Us\\n</td>\n",
       "      <td>Who is DigitalOnUs?  At DigitalOnUs  we not on...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>The Green Organic Dutchman Holdings Ltd</td>\n",
       "      <td>The Green Organic Dutchman is a global leader ...</td>\n",
       "      <td>Mississauga</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>750</th>\n",
       "      <td>Principal Clinical Data Scientist Lead</td>\n",
       "      <td>PRA Health Sciences\\n</td>\n",
       "      <td>Do you want to watch clinical development chan...</td>\n",
       "      <td>Remote</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>Data Solution Architect</td>\n",
       "      <td>Onix Networking Corp\\n</td>\n",
       "      <td>Summary: Onix Solution Architect helps custome...</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Canadian Tire\\n</td>\n",
       "      <td>The CTC Personalization &amp; Customer Analytics t...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>582</th>\n",
       "      <td>Data Engineer/Analyst - Active Safety &amp; Autono...</td>\n",
       "      <td>General Motors\\n</td>\n",
       "      <td>About GM  There’s never been a more exciting t...</td>\n",
       "      <td>Ontario</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <td>Senior Data Scientist</td>\n",
       "      <td>Interac\\n</td>\n",
       "      <td>At Interac  we design products and solutions t...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>728</th>\n",
       "      <td>Applied Scientist - Machine Learning</td>\n",
       "      <td>Microsoft\\n</td>\n",
       "      <td>Applied Scientist - Machine Learning  The Inte...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>673</th>\n",
       "      <td>Senior Machine Learning Engineer</td>\n",
       "      <td>Clearco</td>\n",
       "      <td>Clearco is a company built by founders for fou...</td>\n",
       "      <td>Ontario</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>664</th>\n",
       "      <td>Specialist, Loyalty Analytics</td>\n",
       "      <td>Cineplex</td>\n",
       "      <td>Company Description  Life at Cineplex: Cineple...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>ScanSource Canada Inc.\\n</td>\n",
       "      <td>Summary:  The Data Engineer will be responsibl...</td>\n",
       "      <td>Mississauga</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>Cloud Data Engineer</td>\n",
       "      <td>apptoza inc</td>\n",
       "      <td>Role: Cloud Data Engineer  Location: Toronto  ...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>753</th>\n",
       "      <td>Senior Data Analyst, Growth</td>\n",
       "      <td>NorthOne</td>\n",
       "      <td>We make small businesses more successful throu...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Job Title  \\\n",
       "852        HEAD OF ANALYTICS & INSIGHTS/DATA SCIENTIST   \n",
       "207                          Machine Learning Engineer   \n",
       "627       Data Engineer / Power BI Developer, Omnia AI   \n",
       "671  Junior Business Intelligence Developer - Data ...   \n",
       "661  Data Scientist Technical Lead (Montreal), Inta...   \n",
       "489         Bio informaticien/Bioinformatics scientist   \n",
       "434                                     Data Scientist   \n",
       "448           Scientifique de données / Data Scientist   \n",
       "634      Data Management Specialist - Immunotoxicology   \n",
       "748                                     Data Scientist   \n",
       "831  Senior Professional Environmental Scientist (G...   \n",
       "396                                     Data Scientist   \n",
       "438                                     Data Scientist   \n",
       "11   Junior Project Scientist / Junior Project Tech...   \n",
       "59                              Data Engineer - 313378   \n",
       "383                                Senior Data Analyst   \n",
       "591                          Data Scientist - 05/12/21   \n",
       "577                      Senior Data Engineer - Remote   \n",
       "341                                     DATA SCIENTIST   \n",
       "355                     Data Science Manager, Omnia AI   \n",
       "416                                      DATA ENGINEER   \n",
       "352                                      Data Engineer   \n",
       "733                           Principal Data Scientist   \n",
       "456                        Data Engineer - Mississauga   \n",
       "708        Product Development Confectionary Scientist   \n",
       "12                          Machine Learning Scientist   \n",
       "208      Data Engineering Intern (Fall 2021) - Toronto   \n",
       "537  Manager, Data Science - Research & Analytics T...   \n",
       "639    Ingénieur de données senior / Sr. Data Engineer   \n",
       "669             Solution Architect (Big Data) - Canada   \n",
       "698  Data Analyst Health Surveillance - BC Centre f...   \n",
       "266                 Optimization Specialist, Analytics   \n",
       "761         Machine Learning Engineer / Data Scientist   \n",
       "399                           Data Engineer, Data Lake   \n",
       "686       Product Manager - Data Science - Toronto Hub   \n",
       "267                                     Data Scientist   \n",
       "726                                      Data Engineer   \n",
       "193                                Cloud Data Engineer   \n",
       "60                                       Data Engineer   \n",
       "750             Principal Clinical Data Scientist Lead   \n",
       "179                            Data Solution Architect   \n",
       "370                                     Data Scientist   \n",
       "582  Data Engineer/Analyst - Active Safety & Autono...   \n",
       "512                              Senior Data Scientist   \n",
       "728               Applied Scientist - Machine Learning   \n",
       "673                   Senior Machine Learning Engineer   \n",
       "664                      Specialist, Loyalty Analytics   \n",
       "176                                      Data Engineer   \n",
       "283                                Cloud Data Engineer   \n",
       "753                        Senior Data Analyst, Growth   \n",
       "\n",
       "                                          Company Name  \\\n",
       "852                              Saje Natural Wellness   \n",
       "207                                              Aviva   \n",
       "627                                         Deloitte\\n   \n",
       "671                                     CAPRiser Group   \n",
       "661                                           Intact\\n   \n",
       "489                                           GenAIz\\n   \n",
       "434                                          Awakedata   \n",
       "448                                        BusPatrol\\n   \n",
       "634                       Charles River Laboratories\\n   \n",
       "748                                               F8th   \n",
       "831                 Madrone Environmental Services Ltd   \n",
       "396                                            Novisto   \n",
       "438                                         Peritus.ai   \n",
       "11                                       SNC-Lavalin\\n   \n",
       "59                                            Procom\\n   \n",
       "383                                                ZTR   \n",
       "591                   Acerta Analytics Solutions Inc\\n   \n",
       "577                                             Koho\\n   \n",
       "341                                            Leger\\n   \n",
       "355                                         Deloitte\\n   \n",
       "416                               The Globe and Mail\\n   \n",
       "352                                 Mariner Partners\\n   \n",
       "733                                      Veeva Systems   \n",
       "456                                         Randstad\\n   \n",
       "708                                Cronos Group Inc.\\n   \n",
       "12                      Amazon Dev Centre Canada ULC\\n   \n",
       "208                                         Coursera\\n   \n",
       "537                                            Tonal\\n   \n",
       "639  McKesson Canada Corporation / La Corporation M...   \n",
       "669                           Wavicle Data Solutions\\n   \n",
       "698                                             PHSA\\n   \n",
       "266                                            Torstar   \n",
       "761                           TerraSense Analytics Ltd   \n",
       "399                                      Altus Group\\n   \n",
       "686                                      Veeva Systems   \n",
       "267                                  Oxford Properties   \n",
       "726                                        Pinterest\\n   \n",
       "193                                    Digital On Us\\n   \n",
       "60             The Green Organic Dutchman Holdings Ltd   \n",
       "750                              PRA Health Sciences\\n   \n",
       "179                             Onix Networking Corp\\n   \n",
       "370                                    Canadian Tire\\n   \n",
       "582                                   General Motors\\n   \n",
       "512                                          Interac\\n   \n",
       "728                                        Microsoft\\n   \n",
       "673                                            Clearco   \n",
       "664                                           Cineplex   \n",
       "176                           ScanSource Canada Inc.\\n   \n",
       "283                                        apptoza inc   \n",
       "753                                           NorthOne   \n",
       "\n",
       "                                       Job Description         Location  \\\n",
       "852  For over 25 years we’ve been helping our commu...        Vancouver   \n",
       "207  About the job Individually we are people  but ...          Markham   \n",
       "627  Job Type: Permanent Primary Location: Vancouve...        Vancouver   \n",
       "671  Our subsidiary is in PropTech and real estate ...          Toronto   \n",
       "661  Who needs insurance? Everybody. That keeps us ...         Montreal   \n",
       "489  Avant le SRAS-CoV-2  un vaccin qui était dével...         Montreal   \n",
       "434  RESPONSIBILITIES: Develop state-of-the-art com...          Burnaby   \n",
       "448  Qui sommes-nous : BusPatrouille est une entrep...         Montreal   \n",
       "634  For 70 years  Charles River employees have wor...       Senneville   \n",
       "748  We're transforming the Cyber Security industry...  Midtown Toronto   \n",
       "831  Senior Professional Environmental Scientist an...       Abbotsford   \n",
       "396  At Novisto  our vision is to be the world's le...         Montreal   \n",
       "438  About Peritus.ai   Peritus is dedicated to bui...         Montreal   \n",
       "11   Junior Project Scientist / Junior Project Tech...    Fort St. John   \n",
       "59   Ingénieur de données  Dans le cadre de ses ent...         Montreal   \n",
       "383  Are you passionate about implementing and main...           London   \n",
       "591  Acerta’s machine learning platforms leverage a...        Kitchener   \n",
       "577  About KOHO KOHO is a quickly scaling FinTech c...           Canada   \n",
       "341  WHO WE ARE  At Leger  we know Canadians. With ...  Midtown Toronto   \n",
       "355  Job Type: Permanent Primary Location: Toronto ...  Midtown Toronto   \n",
       "416  DATA ENGINEER POSITION CODE: 2021-063 LOCATION...  Midtown Toronto   \n",
       "352  Company Description As one of the largest prof...          Halifax   \n",
       "733  Veeva [NYSE: VEEV] is the leader in cloud-base...          Toronto   \n",
       "456  Our client  located in Mississauga Ontario is ...      Mississauga   \n",
       "708  At Cronos Group  we hire talented people who t...          Stayner   \n",
       "12   Bachelor’s degree in Computer Science  Statist...        Vancouver   \n",
       "208  Are you ready to join a team of tech-savvy mus...           Remote   \n",
       "537  Who We Are  Tonal is the smartest home gym and...  Midtown Toronto   \n",
       "639  English version to follow  McKesson touche la ...    Saint-Laurent   \n",
       "669  Wavicle Data Solutions designs and delivers da...           Remote   \n",
       "698  Data Analyst Health Surveillance Epidemiology ...        Vancouver   \n",
       "266  Company Description  Torstar Corporation is a ...          Toronto   \n",
       "761  CATCHY INTRO  Honestly  we’ve been brainstormi...          Kelowna   \n",
       "399  Job Category:  Software  Opportunity Awaits at...  Midtown Toronto   \n",
       "686  Veeva [NYSE: VEEV] is the leader in cloud-base...          Toronto   \n",
       "267  Why join us?  Are you looking to join an innov...          Toronto   \n",
       "726  About Pinterest:  Millions of people across th...  Midtown Toronto   \n",
       "193  Who is DigitalOnUs?  At DigitalOnUs  we not on...  Midtown Toronto   \n",
       "60   The Green Organic Dutchman is a global leader ...      Mississauga   \n",
       "750  Do you want to watch clinical development chan...           Remote   \n",
       "179  Summary: Onix Solution Architect helps custome...           Quebec   \n",
       "370  The CTC Personalization & Customer Analytics t...  Midtown Toronto   \n",
       "582  About GM  There’s never been a more exciting t...          Ontario   \n",
       "512  At Interac  we design products and solutions t...  Midtown Toronto   \n",
       "728  Applied Scientist - Machine Learning  The Inte...        Vancouver   \n",
       "673  Clearco is a company built by founders for fou...          Ontario   \n",
       "664  Company Description  Life at Cineplex: Cineple...          Toronto   \n",
       "176  Summary:  The Data Engineer will be responsibl...      Mississauga   \n",
       "283  Role: Cloud Data Engineer  Location: Toronto  ...  Midtown Toronto   \n",
       "753  We make small businesses more successful throu...          Toronto   \n",
       "\n",
       "     Job Board  \n",
       "852     Indeed  \n",
       "207     Indeed  \n",
       "627  Glassdoor  \n",
       "671     Indeed  \n",
       "661  Glassdoor  \n",
       "489  Glassdoor  \n",
       "434  Glassdoor  \n",
       "448  Glassdoor  \n",
       "634  Glassdoor  \n",
       "748  Glassdoor  \n",
       "831  Glassdoor  \n",
       "396  Glassdoor  \n",
       "438  Glassdoor  \n",
       "11   Glassdoor  \n",
       "59   Glassdoor  \n",
       "383     Indeed  \n",
       "591  Glassdoor  \n",
       "577  Glassdoor  \n",
       "341  Glassdoor  \n",
       "355  Glassdoor  \n",
       "416  Glassdoor  \n",
       "352  Glassdoor  \n",
       "733     Indeed  \n",
       "456  Glassdoor  \n",
       "708  Glassdoor  \n",
       "12   Glassdoor  \n",
       "208  Glassdoor  \n",
       "537  Glassdoor  \n",
       "639  Glassdoor  \n",
       "669  Glassdoor  \n",
       "698  Glassdoor  \n",
       "266     Indeed  \n",
       "761  Glassdoor  \n",
       "399  Glassdoor  \n",
       "686     Indeed  \n",
       "267     Indeed  \n",
       "726  Glassdoor  \n",
       "193  Glassdoor  \n",
       "60   Glassdoor  \n",
       "750  Glassdoor  \n",
       "179  Glassdoor  \n",
       "370  Glassdoor  \n",
       "582  Glassdoor  \n",
       "512  Glassdoor  \n",
       "728  Glassdoor  \n",
       "673     Indeed  \n",
       "664     Indeed  \n",
       "176  Glassdoor  \n",
       "283  Glassdoor  \n",
       "753     Indeed  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import regex as re\n",
    "df['Job Description'] = df['Job Description'].apply(lambda x: re.sub(r'[!;,\\s]', ' ', x))\n",
    "df.sample(50, random_state = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "43a999af",
   "metadata": {},
   "outputs": [],
   "source": [
    "ads = list(df['Job Description'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "621fd9cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "881"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d6110482",
   "metadata": {},
   "outputs": [],
   "source": [
    "def highlight_terms(terms, texts):\n",
    "    for doc in nlp.pipe(texts):\n",
    "        for sentence in set([tok.sent for tok in doc if tok.lower_ in terms]):\n",
    "            text = sentence.text.strip()\n",
    "            markup = re.sub(fr'(?i)\\b({\"|\".join(terms)})\\b', r'<strong>\\1</strong>', text)\n",
    "            display(HTML(markup))\n",
    "            print('-----')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ba36c5c1",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "Requirements:  3+ years in data scripting languages e.g Python Hands-on <strong>experience</strong> in the application of predictive modeling  artificial neural networks"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "time-series analysis  clustering and classification techniques  and other advanced analytical techniques <strong>Experience</strong> using CI/CD tools Good design and problem-solving skills and an ability to innovate and solve challenging technical problems Knowledge of the software development lifecycle and Agile methodologies Excellent organization and time management skills"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "The successful candidate will work closely with the product and engineering team to translate technological and customer requirements into valuable features that ensure a great user <strong>experience</strong>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<strong>Experience</strong> with time-series and operations data is"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "an asset Previous Kaggle <strong>experience</strong> or similar is a plus Ability to work in a fast-paced agile environment <strong>Experience</strong> using Git in a team environment Flexibility to adjust to changing priorities"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Minimum of 3 - 5 years of data science <strong>experience</strong> using python Strong ML and statistics background 2+ years of software development <strong>experience</strong> <strong>Experience</strong> with keras  tensorflow  and sklearn."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Live and Digital fitness <strong>experience</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "+ years’ <strong>experience</strong> in ETL/data modeling  building scalable and reliable data pipelines"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "architecting data stores  supporting complex ETL processes including SLA  performance measurements  and monitoring Hands-on <strong>experience</strong> with leading Cloud platforms  including Azure Cloud Data Solutions and AWS <strong>Experience</strong> with Informatica IICS and Snowflake considered a strong asset <strong>Experience</strong> with message queues  batch and stream processing Excellent SQL skills and <strong>experience</strong> in a DevOps environment <strong>Experience</strong> with Agile development methodologies Detail oriented:"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Equivalent combination of education  training and <strong>experience</strong>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Qualifications High School Diploma or equivalent Req 1 to 2 years’ relevant work <strong>experience</strong>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Bonus Skills  <strong>Experience</strong> with Ariflow"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Strong knowledge of SQL (MySQL  PostgreSQL) and preferably <strong>experience</strong> with NoSQL databases Proficient in at least one programming language and scripting languages (bash  Python  etc.)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<strong>Experience</strong> with various AWS services (EC2  S3"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "5+ years of <strong>experience</strong> in big data engineering Have deep understanding on data engineering principles Hands-on <strong>experience</strong> of large scale and high availability systems"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Collaborate with other groups worldwide to assist in product design  business strategy and user <strong>experience</strong> research  etc."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "BS or MS in Computer Science or equivalent <strong>experience</strong>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Redshift  OLAP <strong>Experience</strong> with Git and CircleCI DevOps skills:"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Shift Technology offers a unique opportunity to brilliant candidates to accelerate their careers in data science  Data scientists work on a broad range of subjects  acquiring a lot of technical and professional <strong>experience</strong> in data science"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "We help insurers fully automate more claims  deliver a great customer <strong>experience</strong> while protecting against risk and accurately identifying suspected fraud  making internal teams more effective and improving financial performance."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "A multi-platform production <strong>experience</strong> with commercial and open-source data mining frameworks like Python  R"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "At the Intact Lab  we use machine learning  data science  software engineering  AI  agility  UX and design thinking to transform the customer <strong>experience</strong> for millions of Canadians."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Your 8 years of <strong>experience</strong> in the field of advanced statistics  data mining and text mining Strong communication"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<strong>Experience</strong> working with large financial institutions."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "They understand the technical aspects of data engineering and analysis but are focused on what the data is saying and how we can use it to provide a more personalized and targeted <strong>experience</strong> for our customers."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Analyze data and developer feedback to understand needs  reduce time to onboard and build on the platform  and improve overall stakeholder <strong>experience</strong>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "What we’re looking for:  Degree in Computer Science or equivalent with 5+ years of data analysis and product management <strong>experience</strong>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "SaaS or cloud-based app product <strong>experience</strong>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Demonstrated <strong>experience</strong> in building or delivering data store/warehouse solutions using ML  NLP  etc."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Hands on data extraction and analytical <strong>experience</strong> with SQL  Google Firebase  Metabase and Tableau."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<strong>Experience</strong> and success in both small startup environments and large matrix organizations."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Nice to have:  <strong>Experience</strong> with Python."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<strong>Experience</strong> exploring new capabilities and technologies to drive innovation."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Python <strong>experience</strong> Hive  Spark"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "We love to turn data into stories - stories about money falling through the cracks  success stories about our products and services  but most importantly stories about our customers and how we can enhance their <strong>experience</strong>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "You have advanced working SQL knowledge and <strong>experience</strong> working with relational databases  query authoring (SQL) as well as working familiarity with a variety of databases"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "NiFi <strong>experience</strong> B2B marketing <strong>Experience</strong> with cloud (GCP AWS or Azure) <strong>Experience</strong> with the Salesforce"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Einstein Data Science / Modeling <strong>experience</strong> or working in a Data Science team  Contract would be until 31st Dec 2021 with a high possibility of extension."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "You have minimum 4-7 years of <strong>experience</strong> in a similar role and/or related education with demonstrated performance"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "Must-have:  4-6 years of <strong>experience</strong> with data engineering skills 4-6 years of hands on <strong>experience</strong> with SQL  Shell Scripting and Python"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<strong>Experience</strong>:  data engineering: 4 years (required)"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "4-6 years of hands on <strong>experience</strong> ingesting data using APIs 4-6 years of hands on <strong>experience</strong> with BI tools / Big data technologies like Hadoop  Spark  Scala  Nice-to-Have:  Knowledge of Dialogflow ES and CX APIs  BigQuery"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----\n"
     ]
    }
   ],
   "source": [
    "highlight_terms(['experience'], ads[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "cc9052d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "matcher = Matcher(nlp.vocab)\n",
    "pattern = [{'POS': 'NOUN', 'OP': '+'}, {'LOWER': 'experience'}]\n",
    "matcher.add('experience_noun', [pattern])\n",
    "\n",
    "pattern = [{'LOWER': 'experience'}, {'POS': 'ADP'}, {'POS': {'IN': ('DET', 'NOUN', 'PROPN')}, 'OP': '+'}]\n",
    "matcher.add('experience_adp', [pattern])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d3daa69e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(12285600890577657150, 117, 119),\n",
       " (1417798585642285709, 239, 242),\n",
       " (1417798585642285709, 239, 243),\n",
       " (12285600890577657150, 265, 267)]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc = nlp(ads[0])\n",
    "matcher(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "112ca49c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_extraction(examples, *extractors):\n",
    "    seen = set()\n",
    "    for doc in nlp.pipe(examples):\n",
    "        doc.ents = filter_spans([Span(doc, start, end, label) for extractor in extractors for label, start, end in extractor(doc)])\n",
    "        for tok in doc:\n",
    "            if tok.lower_ == 'experience':\n",
    "                sentence = tok.sent\n",
    "                if sentence.text in seen:\n",
    "                    continue\n",
    "                seen.update([sentence.text])\n",
    "                if not sentence.ents:\n",
    "                    doc.ents = list(doc.ents) + [Span(doc, tok.i, tok.i+1, 'MISSING')]\n",
    "                displacy.render(sentence, style='ent', options = {'colors': {'MISSING': 'pink',\n",
    "                                                                            'EXPERIENCE': 'lightgreen'}})\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "fd43fd0b",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">The successful candidate will work closely with the product and engineering team to translate technological and customer requirements into valuable features that ensure a great \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    user experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_noun</span>\n",
       "</mark>\n",
       ".   </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Requirements:  3+ years in data scripting languages e.g Python Hands-on \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience in the application\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       " of predictive modeling  artificial neural networks  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">time-series analysis  clustering and classification techniques  and other advanced analytical \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    techniques Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_noun</span>\n",
       "</mark>\n",
       " using CI/CD tools Good design and problem-solving skills and an ability to innovate and solve challenging technical problems Knowledge of the software development lifecycle and Agile methodologies Excellent organization and time management skills</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Minimum of 3 - 5 years of \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    data science experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_noun</span>\n",
       "</mark>\n",
       " using python Strong ML and statistics background 2+ years of \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    software development experience Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_noun</span>\n",
       "</mark>\n",
       " with keras  tensorflow  and sklearn. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience with time\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       "-series and operations data is </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">an asset Previous Kaggle experience or similar is a plus Ability to work in a fast-paced agile \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    environment Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_noun</span>\n",
       "</mark>\n",
       " using Git in a team environment Flexibility to adjust to changing priorities  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Live and Digital \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    fitness experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_noun</span>\n",
       "</mark>\n",
       "  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">+ years’ \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience in ETL\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       "/data modeling  building scalable and reliable data pipelines  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">architecting data stores  supporting complex ETL processes including SLA  performance measurements  and monitoring Hands-on experience with leading Cloud platforms  including Azure Cloud Data Solutions and AWS \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience with Informatica IICS\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       " and Snowflake considered a strong asset \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience with message queues\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       "  batch and stream processing Excellent SQL skills and \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience in a DevOps environment Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       " with Agile development methodologies Detail oriented: </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Qualifications High School Diploma or equivalent Req 1 to 2 years’ relevant \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    work experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_noun</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Equivalent combination of education  training and \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Collaborate with other groups worldwide to assist in product design  business strategy and \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    user experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_noun</span>\n",
       "</mark>\n",
       " research  etc. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">5+ years of \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " in big data engineering Have deep understanding on data engineering principles Hands-on experience of large scale and high availability systems </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Strong knowledge of SQL (MySQL  PostgreSQL) and preferably \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience with NoSQL\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       " databases Proficient in at least one programming language and scripting languages (bash  Python  etc.) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " with various AWS services (EC2  S3  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">BS or MS in Computer Science or equivalent \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       "  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Bonus Skills  \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience with Ariflow\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       "  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Redshift  OLAP \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience with Git\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       " and CircleCI DevOps skills: </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_extraction(ads[:5], matcher)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a83e8e12",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_extractions(examples, *extractors):\n",
    "    # Could use context instead of enumerate\n",
    "    for idx, doc in enumerate(nlp.pipe(examples, batch_size=100, disable=['ner'])):\n",
    "        for ent in filter_spans([Span(doc, start, end, label) for extractor in extractors for label, start, end in extractor(doc)]):\n",
    "            sent = ent.root.sent\n",
    "            yield ent.text, idx, ent.start, ent.end, ent.label_, sent.start, sent.end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2c37ba03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('user experience', 0, 117, 119, 'experience_noun', 92, 121),\n",
       " ('experience in the application', 0, 239, 243, 'experience_adp', 224, 251),\n",
       " ('techniques Experience', 0, 265, 267, 'experience_noun', 251, 304),\n",
       " ('data science experience', 1, 143, 146, 'experience_noun', 136, 169),\n",
       " ('software development experience Experience',\n",
       "  1,\n",
       "  157,\n",
       "  161,\n",
       "  'experience_noun',\n",
       "  136,\n",
       "  169),\n",
       " ('Experience with time', 1, 169, 172, 'experience_adp', 169, 178),\n",
       " ('environment Experience', 1, 197, 199, 'experience_noun', 178, 212),\n",
       " ('fitness experience', 2, 73, 75, 'experience_noun', 70, 76),\n",
       " ('experience in ETL', 2, 399, 402, 'experience_adp', 396, 413),\n",
       " ('Experience with Informatica IICS', 2, 445, 449, 'experience_adp', 413, 481),\n",
       " ('Experience with message queues', 2, 455, 459, 'experience_adp', 413, 481),\n",
       " ('experience in a DevOps environment Experience',\n",
       "  2,\n",
       "  468,\n",
       "  474,\n",
       "  'experience_adp',\n",
       "  413,\n",
       "  481)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(get_extractions(ads[:3], matcher))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "aaf13394",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_df(*extractors, n_max=None, **kwargs):\n",
    "    if n_max is None:\n",
    "        n_max = len(df)\n",
    "    ent_df = pd.DataFrame(list(get_extractions(df[:n_max]['Job Description'], *extractors)),\n",
    "                          columns=['text', 'docidx', 'start', 'end', 'label', 'sent_start', 'sent_end'])\n",
    "    return ent_df.merge(df, how='left', left_on='docidx', right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a7d212db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1 µs, sys: 0 ns, total: 1 µs\n",
      "Wall time: 4.29 µs\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>docidx</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>label</th>\n",
       "      <th>sent_start</th>\n",
       "      <th>sent_end</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Location</th>\n",
       "      <th>Job Board</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>user experience</td>\n",
       "      <td>0</td>\n",
       "      <td>117</td>\n",
       "      <td>119</td>\n",
       "      <td>experience_noun</td>\n",
       "      <td>92</td>\n",
       "      <td>121</td>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Metricsflow Inc</td>\n",
       "      <td>At Metricsflow  we work to shape the future of...</td>\n",
       "      <td>St. John's</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>experience in the application</td>\n",
       "      <td>0</td>\n",
       "      <td>239</td>\n",
       "      <td>243</td>\n",
       "      <td>experience_adp</td>\n",
       "      <td>224</td>\n",
       "      <td>251</td>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Metricsflow Inc</td>\n",
       "      <td>At Metricsflow  we work to shape the future of...</td>\n",
       "      <td>St. John's</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>techniques Experience</td>\n",
       "      <td>0</td>\n",
       "      <td>265</td>\n",
       "      <td>267</td>\n",
       "      <td>experience_noun</td>\n",
       "      <td>251</td>\n",
       "      <td>304</td>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Metricsflow Inc</td>\n",
       "      <td>At Metricsflow  we work to shape the future of...</td>\n",
       "      <td>St. John's</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data science experience</td>\n",
       "      <td>1</td>\n",
       "      <td>143</td>\n",
       "      <td>146</td>\n",
       "      <td>experience_noun</td>\n",
       "      <td>136</td>\n",
       "      <td>169</td>\n",
       "      <td>Data Scientist - 05/12/21</td>\n",
       "      <td>Acerta Analytics Solutions Inc</td>\n",
       "      <td>Acerta’s machine learning platforms leverage a...</td>\n",
       "      <td>Kitchener</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>software development experience Experience</td>\n",
       "      <td>1</td>\n",
       "      <td>157</td>\n",
       "      <td>161</td>\n",
       "      <td>experience_noun</td>\n",
       "      <td>136</td>\n",
       "      <td>169</td>\n",
       "      <td>Data Scientist - 05/12/21</td>\n",
       "      <td>Acerta Analytics Solutions Inc</td>\n",
       "      <td>Acerta’s machine learning platforms leverage a...</td>\n",
       "      <td>Kitchener</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         text  docidx  start  end  \\\n",
       "0                             user experience       0    117  119   \n",
       "1               experience in the application       0    239  243   \n",
       "2                       techniques Experience       0    265  267   \n",
       "3                     data science experience       1    143  146   \n",
       "4  software development experience Experience       1    157  161   \n",
       "\n",
       "             label  sent_start  sent_end                  Job Title  \\\n",
       "0  experience_noun          92       121             Data Scientist   \n",
       "1   experience_adp         224       251             Data Scientist   \n",
       "2  experience_noun         251       304             Data Scientist   \n",
       "3  experience_noun         136       169  Data Scientist - 05/12/21   \n",
       "4  experience_noun         136       169  Data Scientist - 05/12/21   \n",
       "\n",
       "                     Company Name  \\\n",
       "0                 Metricsflow Inc   \n",
       "1                 Metricsflow Inc   \n",
       "2                 Metricsflow Inc   \n",
       "3  Acerta Analytics Solutions Inc   \n",
       "4  Acerta Analytics Solutions Inc   \n",
       "\n",
       "                                     Job Description    Location  Job Board  \n",
       "0  At Metricsflow  we work to shape the future of...  St. John's  Glassdoor  \n",
       "1  At Metricsflow  we work to shape the future of...  St. John's  Glassdoor  \n",
       "2  At Metricsflow  we work to shape the future of...  St. John's  Glassdoor  \n",
       "3  Acerta’s machine learning platforms leverage a...   Kitchener     Indeed  \n",
       "4  Acerta’s machine learning platforms leverage a...   Kitchener     Indeed  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time \n",
    "ent_df = extract_df(matcher, n_max=1000)\n",
    "ent_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "86d7302e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def aggregate_df(df, col=['text']):\n",
    "    return (df\n",
    "            .groupby(col)\n",
    "            .agg(n_company=('Company Name', 'nunique'))\n",
    "            .reset_index()\n",
    "            .sort_values(['n_company'], ascending=False)\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "36c78a9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>n_company</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1369</th>\n",
       "      <td>work experience</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1383</th>\n",
       "      <td>years experience</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1203</th>\n",
       "      <td>industry experience</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619</th>\n",
       "      <td>customer experience</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>793</th>\n",
       "      <td>experience in a</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1359</th>\n",
       "      <td>user experience</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>870</th>\n",
       "      <td>experience in data science</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>Experience with Python</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1066</th>\n",
       "      <td>experience with Python</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>931</th>\n",
       "      <td>experience in the</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            text  n_company\n",
       "1369             work experience         49\n",
       "1383            years experience         45\n",
       "1203         industry experience         34\n",
       "619          customer experience         29\n",
       "793              experience in a         27\n",
       "1359             user experience         22\n",
       "870   experience in data science         20\n",
       "329       Experience with Python         19\n",
       "1066      experience with Python         15\n",
       "931            experience in the         15"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aggregate_df(ent_df).head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "aefee619",
   "metadata": {},
   "outputs": [],
   "source": [
    "def showent(docidx, start, end, label, sent_start, sent_end, **kwargs):\n",
    "    # We don't need to parse it, so just make_doc\n",
    "    doc = nlp.make_doc(ads[docidx])\n",
    "    doc.ents = [Span(doc, start, end, label)]\n",
    "    sent = doc[sent_start:sent_end]\n",
    "    displacy.render(sent, style='ent')\n",
    "    \n",
    "def showent_df(df):\n",
    "    for idx, row in df.iterrows():\n",
    "        showent(**row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "570f0451",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have minimum 4-7 years of \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience in a\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       " similar role and/or related education with demonstrated performance  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">production analytics solutions Experience in Data Science using machine learning frameworks and packages such as Scikitlearn  Keras  PyTorch  TensorFlow Consulting or \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience in a\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       " clientfacing role Experience with discrete event simulation Experience with Big Data technologies such as Hadoop and Spark Experience with visualization platforms such as Tableau and PowerBI Experience with relational databases  SQL and data modeling </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">We are looking for a Senior Data Scientist with a minimum of 5 years of \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience in a\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       " related field to join our team in the Vancouver office. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">At least 2 years of \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience in a\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       " similar role or function Skills with: Python  SQL and Docker AWS familiarity  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have minimum 4-7 years of \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience in a\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">experience_adp</span>\n",
       "</mark>\n",
       " similar role and/or related education with demonstrated performance Great-to-haves Prior telecommunications expertise </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showent_df(ent_df.query('text == \"experience in a\"').head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "14a8b08d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_noun_phrase_experience(doc):\n",
    "    for np in doc.noun_chunks:\n",
    "        if np[-1].lower_ == 'experience':\n",
    "            if len(np) > 1:\n",
    "                yield 'EXPERIENCE', np[0].i, np[-1].i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a287f1fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">The successful candidate will work closely with the product and engineering team to translate technological and customer requirements into valuable features that ensure \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    a great user\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " experience.   </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Requirements:  3+ years in data scripting languages \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    e.g Python Hands-on\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " experience in the application of predictive modeling  artificial neural networks  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">time-series analysis  clustering and classification techniques  and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    other advanced analytical techniques\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " Experience using CI/CD tools Good design and problem-solving skills and an ability to innovate and solve challenging technical problems Knowledge of the software development lifecycle and Agile methodologies Excellent organization and time management skills</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Minimum of 3 - 5 years of \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    data science\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " experience using python Strong ML and statistics background 2+ years of \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    software development experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " Experience with keras  tensorflow  and sklearn. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " with time-series and operations data is </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    an asset Previous Kaggle\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " experience or similar is a plus Ability to work in \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    a fast-paced agile environment\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " Experience using Git in a team environment Flexibility to adjust to changing priorities  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Live and Digital fitness\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " experience  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">+ \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    years’\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " experience in ETL/data modeling  building scalable and reliable data pipelines  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">architecting data stores  supporting complex ETL processes including SLA  performance measurements  and monitoring \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Hands-on\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " experience with leading Cloud platforms  including Azure Cloud Data Solutions and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    AWS\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " Experience with Informatica IICS and Snowflake considered \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    a strong asset\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " Experience with message queues  batch and stream processing Excellent SQL skills and experience in \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    a DevOps environment\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " Experience with Agile development methodologies Detail oriented: </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Qualifications High School Diploma or \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    equivalent Req 1 to 2 years’ relevant work\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " experience. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Equivalent combination of education  training and \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Collaborate with other groups worldwide to assist in product design  business strategy and user \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " research  etc. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">5+ years of experience in big data engineering Have deep understanding on data engineering principles \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Hands-on\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " experience of large scale and high availability systems </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Strong knowledge of SQL (MySQL  PostgreSQL) and preferably \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " with NoSQL databases Proficient in at least one programming language and scripting languages (bash  Python  etc.) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " with various AWS services (EC2  S3  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">BS or MS in Computer Science or \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    equivalent\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " experience  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Bonus Skills  \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " with Ariflow  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Redshift  OLAP\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " Experience with Git and CircleCI DevOps skills: </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_extraction(ads[:5], extract_noun_phrase_experience)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "cc8b16d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1 µs, sys: 0 ns, total: 1 µs\n",
      "Wall time: 3.1 µs\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "ent_df = extract_df(extract_noun_phrase_experience, n_max=50000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "32eb6e39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>n_company</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>897</th>\n",
       "      <td>hands-on</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>925</th>\n",
       "      <td>industry</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>853</th>\n",
       "      <td>equivalent</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>287</th>\n",
       "      <td>Hands-on</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1083</th>\n",
       "      <td>relevant</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>253</th>\n",
       "      <td>Extensive</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1273</th>\n",
       "      <td>years</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>425</th>\n",
       "      <td>Proven</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875</th>\n",
       "      <td>extensive</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1265</th>\n",
       "      <td>work</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Demonstrated</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1044</th>\n",
       "      <td>professional</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>404</th>\n",
       "      <td>Prior</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>527</th>\n",
       "      <td>Strong</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1172</th>\n",
       "      <td>the</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1282</th>\n",
       "      <td>your</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>Previous</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>773</th>\n",
       "      <td>customer</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>)</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420</th>\n",
       "      <td>Programming</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1291</th>\n",
       "      <td>·</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>5+ years</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>3+ years</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>3-5 years</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>621</th>\n",
       "      <td>a plus</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>677</th>\n",
       "      <td>an asset</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1104</th>\n",
       "      <td>relevant work</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419</th>\n",
       "      <td>Professional</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1199</th>\n",
       "      <td>the customer</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>866</th>\n",
       "      <td>equivalent work</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1020</th>\n",
       "      <td>preferred</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>SQL</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389</th>\n",
       "      <td>Practical</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1093</th>\n",
       "      <td>relevant industry</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>781</th>\n",
       "      <td>data</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>548</th>\n",
       "      <td>The</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1070</th>\n",
       "      <td>related</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1233</th>\n",
       "      <td>the user</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1013</th>\n",
       "      <td>practical</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1124</th>\n",
       "      <td>significant</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1025</th>\n",
       "      <td>previous</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>Significant</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>Python</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1256</th>\n",
       "      <td>user</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>513</th>\n",
       "      <td>Some</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1048</th>\n",
       "      <td>programming</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1029</th>\n",
       "      <td>prior</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1234</th>\n",
       "      <td>their</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1269</th>\n",
       "      <td>working</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259</th>\n",
       "      <td>Friday</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   text  n_company\n",
       "897            hands-on         29\n",
       "925            industry         28\n",
       "853          equivalent         25\n",
       "287            Hands-on         23\n",
       "1083           relevant         22\n",
       "253           Extensive         18\n",
       "1273              years         17\n",
       "425              Proven         17\n",
       "875           extensive         16\n",
       "1265               work         16\n",
       "218        Demonstrated         16\n",
       "1044       professional         15\n",
       "404               Prior         15\n",
       "527              Strong         15\n",
       "1172                the         14\n",
       "1282               your         14\n",
       "397            Previous         14\n",
       "773            customer         13\n",
       "13                    )         12\n",
       "420         Programming         11\n",
       "1291                  ·         11\n",
       "70             5+ years         10\n",
       "54             3+ years         10\n",
       "58            3-5 years         10\n",
       "621              a plus         10\n",
       "677            an asset          9\n",
       "1104      relevant work          9\n",
       "419        Professional          9\n",
       "1199       the customer          9\n",
       "866     equivalent work          9\n",
       "1020          preferred          9\n",
       "489                 SQL          8\n",
       "389           Practical          8\n",
       "1093  relevant industry          8\n",
       "781                data          8\n",
       "548                 The          8\n",
       "1070            related          8\n",
       "1233           the user          8\n",
       "1013          practical          8\n",
       "1124        significant          8\n",
       "1025           previous          8\n",
       "507         Significant          7\n",
       "437              Python          7\n",
       "1256               user          7\n",
       "513                Some          7\n",
       "1048        programming          6\n",
       "1029              prior          6\n",
       "1234              their          6\n",
       "1269            working          6\n",
       "259              Friday          6"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aggregate_df(ent_df).head(50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "decacb25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\\\b(?:previous|prior|following|recent|the above|past|proven|demonstrable|demonstrated|relevant|significant|practical|essential|equivalent|desirable|required|considerable|similar|working|specific|qualified|direct|hands on|handson|hands-onstrong|solid|good|substantial|excellent|the right|valuable|invaluable|some|any|none|much|extensive|no|more|your|their|great|years|months)\\\\b'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experience_qualifiers = ['previous', 'prior', 'following', 'recent', 'the above', 'past',\n",
    "                         \n",
    "                         'proven', 'demonstrable', 'demonstrated', 'relevant', 'significant', 'practical',\n",
    "                         'essential', 'equivalent', 'desirable', 'required', 'considerable', 'similar',\n",
    "                         'working', 'specific', 'qualified', 'direct', 'hands on', 'handson', 'hands-on'\n",
    "                         \n",
    "                         'strong', 'solid', 'good', 'substantial', 'excellent', 'the right', 'valuable', 'invaluable',\n",
    "                         \n",
    "                         'some', 'any', 'none', 'much', 'extensive', 'no', 'more',\n",
    "                         'your', 'their', 'great',\n",
    "                         'years', 'months',\n",
    "                        ]\n",
    "\n",
    "stopwords = ['a', 'an', '*', '**', '.', 'this', 'the', ':', 'Skills']\n",
    "\n",
    "experience_qualifier_pattern = rf'\\b(?:{\"|\".join(experience_qualifiers)})\\b'\n",
    "\n",
    "experience_qualifier_pattern"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "619e8a18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>n_company</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>665</th>\n",
       "      <td>hands-on</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>690</th>\n",
       "      <td>industry</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>Hands-on</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>946</th>\n",
       "      <td>work</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>791</th>\n",
       "      <td>professional</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>365</th>\n",
       "      <td>Strong</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>573</th>\n",
       "      <td>customer</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>)</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>Programming</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>953</th>\n",
       "      <td>·</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>441</th>\n",
       "      <td>a plus</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>894</th>\n",
       "      <td>the customer</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>288</th>\n",
       "      <td>Professional</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>491</th>\n",
       "      <td>an asset</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>775</th>\n",
       "      <td>preferred</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>581</th>\n",
       "      <td>data</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>SQL</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>The</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811</th>\n",
       "      <td>related</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>924</th>\n",
       "      <td>the user</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298</th>\n",
       "      <td>Python</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>939</th>\n",
       "      <td>user</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>Friday</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>795</th>\n",
       "      <td>programming</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>376</th>\n",
       "      <td>Tableau</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>541</th>\n",
       "      <td>business</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>Data Science</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <td>analytics</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>796</th>\n",
       "      <td>progressive</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>AWS</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358</th>\n",
       "      <td>Spark</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>Work</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>639</th>\n",
       "      <td>engineering</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>851</th>\n",
       "      <td>strong</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>885</th>\n",
       "      <td>the best</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>611</th>\n",
       "      <td>deep</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>582</th>\n",
       "      <td>data analysis</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>Databricks</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>618</th>\n",
       "      <td>design</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>663</th>\n",
       "      <td>global</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>672</th>\n",
       "      <td>hands-on work</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>643</th>\n",
       "      <td>etc</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>722</th>\n",
       "      <td>modeling</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>Scala</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>622</th>\n",
       "      <td>development</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>Applied</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>Data integration</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>442</th>\n",
       "      <td>a plus</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>617</th>\n",
       "      <td>deployment</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>advanced analytics</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   text  n_company\n",
       "665            hands-on         29\n",
       "690            industry         28\n",
       "191            Hands-on         23\n",
       "946                work         16\n",
       "791        professional         15\n",
       "365              Strong         15\n",
       "573            customer         13\n",
       "13                    )         12\n",
       "289         Programming         11\n",
       "953                   ·         11\n",
       "441              a plus         10\n",
       "894        the customer          9\n",
       "288        Professional          9\n",
       "491            an asset          9\n",
       "775           preferred          9\n",
       "581                data          8\n",
       "338                 SQL          8\n",
       "383                 The          8\n",
       "811             related          8\n",
       "924            the user          8\n",
       "298              Python          7\n",
       "939                user          7\n",
       "165              Friday          6\n",
       "795         programming          6\n",
       "376             Tableau          5\n",
       "541            business          4\n",
       "119        Data Science          4\n",
       "512           analytics          4\n",
       "796         progressive          4\n",
       "35                  AWS          4\n",
       "358               Spark          4\n",
       "396                Work          4\n",
       "639         engineering          4\n",
       "851              strong          4\n",
       "885            the best          4\n",
       "611                deep          3\n",
       "582       data analysis          3\n",
       "130          Databricks          3\n",
       "618              design          3\n",
       "663              global          3\n",
       "672       hands-on work          3\n",
       "643                 etc          3\n",
       "722            modeling          3\n",
       "352               Scala          3\n",
       "622         development          3\n",
       "60              Applied          3\n",
       "124    Data integration          3\n",
       "442            a plus            3\n",
       "617          deployment          3\n",
       "478  advanced analytics          3"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aggregate_df(ent_df[(~ent_df.text.str.lower().str.contains(experience_qualifier_pattern)) & # Not a qualifier\n",
    "                     ~ent_df.text.isin(stopwords)]).head(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "83ba7d6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_adp_experience(doc, label='EXPERIENCE'):\n",
    "    for tok in doc:\n",
    "        if tok.lower_ == 'experience':\n",
    "            for child in tok.rights:\n",
    "                if child.dep_ == 'prep':\n",
    "                    for obj in child.children:\n",
    "                        if obj.dep_ == 'pobj':\n",
    "                            yield label, obj.left_edge.i, obj.i+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7ffffed9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Job Benefits Profit-sharing  distributed 3 times a year Frequent promotions 3 weeks vacation and paid sick days Happy Hour every Friday Extended health benefits Continued education allowance Annual fitness allowance Work from anywhere in the world Join a bootstrapped  product-focused  &amp; customer-oriented team Minimum 3+ years of data experience  ideally with SaaS Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    MySQL\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " &amp; BigQuery Experience creating dashboards in a BI tool Experience creating workflows Experience using Tray.io (nice to have) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">The ideal candidate is a junior environmental engineering professional or recent graduate with an environmental engineering degree or diploma with some previous relevant \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " which would be an asset.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Previous environmental or consulting work \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " is preferred. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">2+ years of non-internship professional experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    machine\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " learning  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">2+ years of experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Python\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Are you interested in using the latest advances in machine learning  computer vision  and augmented reality to build online customer experiences for Beauty products that can equal or even surpass an in-store \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       "?  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You will have an opportunity to revolutionize the customer shopping experience across \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    the world's most extensive catalog\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " of beauty products. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">scientists  and the broader Amazon tech community to build solutions that enhance the beauty shopping \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " across all surfaces  including desktop  mobile devices  and other Amazon devices.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">product managers  and designers who drive technological innovation to improve customer \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " applying various machine learning techniques  and understanding the key parameters that affect their performance. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " partnering with engineering teams to build and test production systems. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Our ideal candidate should have: Expertise in R and other statistical programming languages (3+ years) Experience in \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Frequentist and Bayesian Statistical methods\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " (2+ years) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " working with Machine Learning algorithms  Probabilistic Models  and/or other statistical modelling approaches (2+ years) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    modern R packages\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " such as dplyr  ggplot2  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">etc Experience in \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    front-end R technologies\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " for data products such as ShinyR  Flexdashboard Ability to write complex SQL queries Working knowledge of software engineering fundamentals and workflows Experience working with container orchestration platforms such as Kubernetes or Docker Swarm Cloud Computing Experience Authority to legally work in Canada </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ": </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">We love to turn data into stories - stories about money falling through the cracks  success stories about our products and services  but most importantly stories about our customers and how we can enhance their \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    agile ways\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " of working and a bias for action to break down barriers to get results fast with a test and learn mindset </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " mapping various customer data using ‘fuzzy matching' to join disconnected data sets You have proven record of implementing business rules  optimizing processes and performing automated data quality audits Strong analytic skills related to working with unstructured datasets. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You can influence team members and work on projects collaboratively Must-haves Advanced working SQL knowledge and \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " working with relational databases  query authoring (SQL) as well as working familiarity with a variety of databases </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Tableau \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">(computer science  engineering  statistics  data science) and/or Masters of Business Administration (MBA) degree 3+ years of progressive and relevant work \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " Great-to-haves </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Python \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " Hive  Spark  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">NiFi experience B2B marketing Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    cloud (GCP AWS or Azure) Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    the Salesforce\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Einstein Data Science / Modeling \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " or working in a Data Science team Who is TELUS? </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_extraction(ads[10:15], extract_adp_experience)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f90762b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_adp_experience_2(doc):\n",
    "    for np in doc.noun_chunks:\n",
    "        start_tok = np[0].i\n",
    "        if start_tok >= 2 and doc[start_tok - 2].lower_ == 'experience' and doc[start_tok - 1].pos_ == 'ADP':\n",
    "            yield 'EXPERIENCE', start_tok, start_tok + len(np)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "efcdcdd2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Job Benefits Profit-sharing  distributed 3 times a year Frequent promotions 3 weeks vacation and paid sick days Happy Hour every Friday Extended health benefits Continued education allowance Annual fitness allowance Work from anywhere in the world Join a bootstrapped  product-focused  &amp; customer-oriented team Minimum 3+ years of data experience  ideally with SaaS Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    MySQL\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " &amp; BigQuery Experience creating dashboards in a BI tool Experience creating workflows Experience using Tray.io (nice to have) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">The ideal candidate is a junior environmental engineering professional or recent graduate with an environmental engineering degree or diploma with some previous relevant \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " which would be an asset.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Previous environmental or consulting work \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " is preferred. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">2+ years of non-internship professional experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    machine\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " learning  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">2+ years of experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Python\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Are you interested in using the latest advances in machine learning  computer vision  and augmented reality to build online customer experiences for Beauty products that can equal or even surpass an in-store \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       "?  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You will have an opportunity to revolutionize the customer shopping experience across \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    the world's most extensive catalog\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " of beauty products. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">scientists  and the broader Amazon tech community to build solutions that enhance the beauty shopping experience across \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    all surfaces\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  including desktop  mobile devices  and other Amazon devices.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">product managers  and designers who drive technological innovation to improve customer \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " applying various machine learning techniques  and understanding the key parameters that affect their performance. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " partnering with engineering teams to build and test production systems. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Our ideal candidate should have: Expertise in R and other statistical programming languages (3+ years) Experience in \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Frequentist and Bayesian Statistical methods\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " (2+ years) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " working with Machine Learning algorithms  Probabilistic Models  and/or other statistical modelling approaches (2+ years) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    modern R packages\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " such as dplyr  ggplot2  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">etc Experience in \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    front-end R technologies\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " for data products such as ShinyR  Flexdashboard Ability to write complex SQL queries Working knowledge of software engineering fundamentals and workflows Experience working with container orchestration platforms such as Kubernetes or Docker Swarm Cloud Computing Experience Authority to legally work in Canada </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ": </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">We love to turn data into stories - stories about money falling through the cracks  success stories about our products and services  but most importantly stories about our customers and how we can enhance their \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    agile ways\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " of working and a bias for action to break down barriers to get results fast with a test and learn mindset </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " mapping various customer data using ‘fuzzy matching' to join disconnected data sets You have proven record of implementing business rules  optimizing processes and performing automated data quality audits Strong analytic skills related to working with unstructured datasets. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You can influence team members and work on projects collaboratively Must-haves Advanced working SQL knowledge and \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " working with relational databases  query authoring (SQL) as well as working familiarity with a variety of databases </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Tableau \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">(computer science  engineering  statistics  data science) and/or Masters of Business Administration (MBA) degree 3+ years of progressive and relevant work \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " Great-to-haves </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Python \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " Hive  Spark  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">NiFi experience B2B marketing Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    cloud (GCP AWS or Azure) Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    the Salesforce\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Einstein Data Science / Modeling \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " or working in a Data Science team Who is TELUS? </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_extraction(ads[10:15], extract_adp_experience_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "22c76778",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1e+03 ns, sys: 0 ns, total: 1e+03 ns\n",
      "Wall time: 3.1 µs\n"
     ]
    }
   ],
   "source": [
    "%time \n",
    "ent_adp_df = extract_df(extract_adp_experience, n_max=50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c309f579",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2 µs, sys: 0 ns, total: 2 µs\n",
      "Wall time: 3.1 µs\n"
     ]
    }
   ],
   "source": [
    "%time \n",
    "ent_adp_df = extract_df(extract_adp_experience_2, n_max=50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "cee8f735",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1e+03 ns, sys: 0 ns, total: 1e+03 ns\n",
      "Wall time: 2.86 µs\n"
     ]
    }
   ],
   "source": [
    "%time \n",
    "ent_adp_df = extract_df(extract_adp_experience, n_max=50000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2f33aff9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>n_company</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>302</th>\n",
       "      <td>Python</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>706</th>\n",
       "      <td>data science</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>673</th>\n",
       "      <td>data</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>328</th>\n",
       "      <td>SQL</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>561</th>\n",
       "      <td>big data tools</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>718</th>\n",
       "      <td>data visualization</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>405</th>\n",
       "      <td>a Data Scientist</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>AWS</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>676</th>\n",
       "      <td>data analysis</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>560</th>\n",
       "      <td>big data technologies</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1047</th>\n",
       "      <td>relational databases</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>358</th>\n",
       "      <td>Spark</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>975</th>\n",
       "      <td>our customers</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>699</th>\n",
       "      <td>data pipeline</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1014</th>\n",
       "      <td>programming languages</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1086</th>\n",
       "      <td>software development</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>721</th>\n",
       "      <td>data visualization tools</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>682</th>\n",
       "      <td>data engineering</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>877</th>\n",
       "      <td>machine</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>753</th>\n",
       "      <td>development</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>678</th>\n",
       "      <td>data analytics</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1042</th>\n",
       "      <td>relational SQL</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1107</th>\n",
       "      <td>stream-processing systems</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>Big Data</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>958</th>\n",
       "      <td>one</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369</th>\n",
       "      <td>Tableau</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>878</th>\n",
       "      <td>machine learning</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>Data Science</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619</th>\n",
       "      <td>cloud services</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>Docker</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1022</th>\n",
       "      <td>python</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1153</th>\n",
       "      <td>the application</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AI</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>748</th>\n",
       "      <td>design</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218</th>\n",
       "      <td>Java</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>426</th>\n",
       "      <td>a data analyst</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>925</th>\n",
       "      <td>modern programming languages</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>203</th>\n",
       "      <td>Hadoop</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>Git</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>308</th>\n",
       "      <td>R</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1025</th>\n",
       "      <td>quantitative analysis</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>any</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>538</th>\n",
       "      <td>at least one programming language</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>733</th>\n",
       "      <td>databases</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Azure</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401</th>\n",
       "      <td>a Data Engineer</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>Databricks</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1055</th>\n",
       "      <td>retail</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>531</th>\n",
       "      <td>applications</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   text  n_company\n",
       "302                              Python         48\n",
       "706                        data science         25\n",
       "673                                data         16\n",
       "328                                 SQL         14\n",
       "244                    Machine Learning         13\n",
       "561                      big data tools         11\n",
       "718                  data visualization         10\n",
       "405                    a Data Scientist          9\n",
       "10                                  AWS          9\n",
       "676                       data analysis          8\n",
       "560               big data technologies          8\n",
       "1047               relational databases          8\n",
       "358                               Spark          7\n",
       "975                       our customers          7\n",
       "699                       data pipeline          7\n",
       "1014              programming languages          7\n",
       "1086               software development          7\n",
       "721            data visualization tools          6\n",
       "682                    data engineering          6\n",
       "877                             machine          6\n",
       "753                         development          6\n",
       "678                      data analytics          6\n",
       "1042                     relational SQL          6\n",
       "1107          stream-processing systems          5\n",
       "63                             Big Data          5\n",
       "958                                 one          5\n",
       "369                             Tableau          5\n",
       "878                    machine learning          5\n",
       "125                        Data Science          5\n",
       "619                      cloud services          5\n",
       "150                              Docker          5\n",
       "1022                             python          5\n",
       "1153                    the application          5\n",
       "4                                    AI          5\n",
       "748                              design          5\n",
       "218                                Java          5\n",
       "426                      a data analyst          4\n",
       "925        modern programming languages          4\n",
       "203                              Hadoop          4\n",
       "186                                 Git          4\n",
       "308                                   R          4\n",
       "1025              quantitative analysis          4\n",
       "524                                 any          4\n",
       "538   at least one programming language          4\n",
       "733                           databases          4\n",
       "48                                Azure          4\n",
       "401                     a Data Engineer          4\n",
       "140                          Databricks          4\n",
       "1055                             retail          4\n",
       "531                        applications          4"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aggregate_df(ent_adp_df).head(50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e323259a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Experience in developing and debugging in C/C++  Python  or C# | Expérience de développement et de débogage en \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    C\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showent_df(ent_adp_df.query(\"text=='C'\").head(5))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "97855a3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">2+ years of experience with \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    R\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " or Python and SQL. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Qualifications 5+ years work experience as a Data Scientist Experience with \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    R\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " or Python and SQL Experience with one or more of the following technologies</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Demonstrated  hands-on experience with \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    R\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  Python  or similar data science tools.    </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Our ideal candidate should have: Degree/Diploma in Computer Science/Software Engineering/Statistics or equivalent 3+ years of relevant experience with \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    R\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " or Python 3+ years of experience with Machine Learning algorithms and Probabilistic Modelling Strong background in statistics  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Our ideal candidate should have:  Degree/Diploma in Computer Science/Software Engineering/Statistics or equivalent 3+ years of relevant experience with \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    R\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " or Python </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showent_df(ent_adp_df.query(\"text=='R'\").head(5))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "9dbf1462",
   "metadata": {},
   "outputs": [],
   "source": [
    "def highlight_text_context(terms, texts, n_before=1, n_after=2):\n",
    "    context = []\n",
    "    for doc in nlp.pipe(texts):\n",
    "        sentences = list(doc.sents)\n",
    "        idxs = [i for i, sent in enumerate(sentences) if any(term in sent.text.lower() for term in terms)]\n",
    "        \n",
    "        for idx in idxs:\n",
    "            before = ''.join(sent.text for sent in sentences[max(idx-n_before, 0):idx])\n",
    "            after = ''.join(sent.text for sent in sentences[idx+1:min(idx+n_before+1, len(sentences))])\n",
    "            text = sentences[idx].text\n",
    "            markup = re.sub(fr'(?i)\\b({\"|\".join(terms)})\\b', r'<strong>\\1</strong>',\n",
    "                                 f'<span style=\"color:blue\">{text}</span>')\n",
    "            display(HTML(before + markup + after))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "0218fbcc",
   "metadata": {},
   "outputs": [],
   "source": [
    "terms = ['experience']\n",
    "\n",
    "for _, q in ent_adp_df.query(\"text=='a'\").head(7).iterrows():\n",
    "    doc = nlp(q.FullDescription)\n",
    "    if q.sent_start > 0:\n",
    "        prev_sent = doc[q.sent_start - 1].sent.text\n",
    "    else:\n",
    "        prev_sent = ''\n",
    "    \n",
    "    if q.sent_end < len(doc):\n",
    "        next_sent = doc[q.sent_end].sent.text\n",
    "    else:\n",
    "        next_sent = ''\n",
    "        \n",
    "    text = doc[q.sent_start:q.sent_end].text\n",
    "    markup = re.sub(fr'(?i)\\b({\"|\".join(terms)})\\b', r'<strong>\\1</strong>',\n",
    "                     f'<span style=\"color:blue\">{text}</span>')\n",
    "    display(HTML(prev_sent + markup + next_sent))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "e3cfcd77",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_left_span(tok, label='', include=True):\n",
    "    offset = 1 if include else 0\n",
    "    idx = tok.i\n",
    "    while idx > tok.left_edge.i:\n",
    "        if tok.doc[idx - 1].pos_ in ('NOUN', 'PROPN', 'ADJ', 'X'):\n",
    "            idx -= 1\n",
    "        else:\n",
    "            break\n",
    "    return label, idx, tok.i+offset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "31e4c860",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_conjugations(tok):\n",
    "    new = [tok]\n",
    "    while new:\n",
    "        tok = new.pop()\n",
    "        yield tok\n",
    "        for child in tok.children:\n",
    "            if child.dep_ == 'conj':\n",
    "                new.append(child)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "8a14976c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#old\n",
    "EXP_TERMS = ['experience']\n",
    "def extract_adp_conj_experience(doc, label='EXPERIENCE'):\n",
    "    for tok in doc:\n",
    "        if tok.lower_ in EXP_TERMS:\n",
    "            for child in tok.rights:\n",
    "                if child.dep_ == 'prep':\n",
    "                    for obj in child.children:\n",
    "                        if obj.dep_ == 'pobj':\n",
    "                            for conj in get_conjugations(obj):\n",
    "                                yield get_left_span(conj, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "055c9de2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Job Benefits Profit-sharing  distributed 3 times a year Frequent promotions 3 weeks vacation and paid sick days Happy Hour every Friday Extended health benefits Continued education allowance Annual fitness allowance Work from anywhere in the world Join a bootstrapped  product-focused  &amp; customer-oriented team Minimum 3+ years of data experience  ideally with SaaS Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    MySQL\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " &amp; \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    BigQuery Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " creating dashboards in a BI tool Experience creating workflows Experience using Tray.io (nice to have) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">The ideal candidate is a junior environmental engineering professional or recent graduate with an environmental engineering degree or diploma with some previous relevant \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " which would be an asset.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Previous environmental or consulting work \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " is preferred. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">2+ years of non-internship professional experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    machine\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " learning  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">2+ years of experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Python\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Are you interested in using the latest advances in machine learning  computer vision  and augmented reality to build online customer experiences for Beauty products that can equal or even surpass an in-store \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       "?  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You will have an opportunity to revolutionize the customer shopping experience across the world's most \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    extensive catalog\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " of beauty products. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">scientists  and the broader Amazon tech community to build solutions that enhance the beauty shopping \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " across all surfaces  including desktop  mobile devices  and other Amazon devices.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">product managers  and designers who drive technological innovation to improve customer \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " applying various machine learning techniques  and understanding the key parameters that affect their performance. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " partnering with engineering teams to build and test production systems. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Our ideal candidate should have: Expertise in R and other statistical programming languages (3+ years) Experience in Frequentist and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Bayesian Statistical methods\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " (2+ years) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " working with Machine Learning algorithms  Probabilistic Models  and/or other statistical modelling approaches (2+ years) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    modern R packages\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " such as dplyr  ggplot2  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">etc Experience in front-\n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    end R technologies\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " for data products such as ShinyR  Flexdashboard Ability to write complex SQL queries Working knowledge of software engineering fundamentals and workflows Experience working with container orchestration platforms such as Kubernetes or Docker Swarm Cloud Computing Experience Authority to legally work in Canada </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ": </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">We love to turn data into stories - stories about money falling through the cracks  success stories about our products and services  but most importantly stories about our customers and how we can enhance their \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    agile ways\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " of working and a \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    bias\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " for action to break down barriers to get results fast with a test and learn mindset </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " mapping various customer data using ‘fuzzy matching' to join disconnected data sets You have proven record of implementing business rules  optimizing processes and performing automated data quality audits Strong analytic skills related to working with unstructured datasets. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You can influence team members and work on projects collaboratively Must-haves Advanced working SQL knowledge and \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " working with relational databases  query authoring (SQL) as well as working familiarity with a variety of databases </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Tableau \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">(computer science  engineering  statistics  data science) and/or Masters of Business Administration (MBA) degree 3+ years of progressive and relevant work \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " Great-to-haves </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Python \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " Hive  Spark  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">NiFi experience B2B marketing Experience with cloud (GCP AWS or Azure) \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " with the \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Salesforce\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Einstein Data Science / Modeling \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " or working in a Data Science team Who is TELUS? </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_extraction(ads[10:15], extract_adp_conj_experience)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "73411223",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">The successful candidate will work closely with the product and engineering team to translate technological and customer requirements into valuable features that ensure a great user \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ".   </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Requirements:  3+ years in data scripting languages e.g Python Hands-on experience in the \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    application\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " of predictive modeling  artificial neural networks  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">time-series analysis  clustering and classification techniques  and other advanced analytical techniques \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " using CI/CD tools Good design and problem-solving skills and an ability to innovate and solve challenging technical problems Knowledge of the software development lifecycle and Agile methodologies Excellent organization and time management skills</div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Minimum of 3 - 5 years of data science experience using python Strong ML and statistics background 2+ years of software development experience Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    keras\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  tensorflow  and sklearn. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Experience with time-series and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    operations data\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " is </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">an asset Previous Kaggle \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " or similar is a plus Ability to work in a fast-paced agile environment Experience using Git in a team environment Flexibility to adjust to changing priorities  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Live and Digital fitness \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       "  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">+ years’ experience in ETL/\n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    data\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " modeling  building scalable and reliable data pipelines  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">architecting data stores  supporting complex ETL processes including SLA  performance measurements  and monitoring Hands-on experience with leading \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Cloud platforms\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  including \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Azure Cloud Data Solutions\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    AWS Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Informatica IICS\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Snowflake\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " considered a strong asset Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    message queues\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  batch and stream processing Excellent SQL skills and experience in a \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    DevOps environment Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Agile development methodologies\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " Detail oriented: </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Qualifications High School Diploma or equivalent Req 1 to 2 years’ relevant work \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Equivalent combination of education  training and \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Collaborate with other groups worldwide to assist in product design  business strategy and user \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " research  etc. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">5+ years of experience in \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    big data engineering\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " Have deep understanding on data engineering principles Hands-on experience of \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    large scale\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    high availability systems\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Strong knowledge of SQL (MySQL  PostgreSQL) and preferably experience with NoSQL databases \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Proficient\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " in at least one \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    programming language\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    scripting languages\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " (bash  Python  etc.) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    various AWS services\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " (EC2  S3  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">BS or MS in Computer Science or equivalent \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       "  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Bonus Skills  Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Ariflow\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Redshift  OLAP Experience with Git and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    CircleCI DevOps skills\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       ": </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">We help insurers fully automate more claims  deliver a great customer \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " while protecting against risk and accurately identifying suspected fraud  making internal teams more effective and improving financial performance.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Shift Technology offers a unique opportunity to brilliant candidates to accelerate their careers in data science  Data scientists work on a broad range of subjects  acquiring a lot of technical and professional experience in \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    data science\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">At the Intact Lab  we use machine learning  data science  software engineering  AI  agility  UX and design thinking to transform the customer \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " for millions of Canadians. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Your 8 years of experience in the \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    field\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " of advanced statistics  data mining and text mining Strong communication  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">A multi-platform production experience with commercial and open-\n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    source data mining frameworks\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " like Python  R  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">They understand the technical aspects of data engineering and analysis but are focused on what the data is saying and how we can use it to provide a more personalized and targeted experience for our \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    customers\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       ".  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Analyze data and developer feedback to understand needs  reduce time to onboard and build on the platform  and improve overall stakeholder \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">What we’re looking for:  Degree in Computer Science or equivalent with 5+ years of data analysis and product management \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Hands on data extraction and analytical \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " with SQL  Google Firebase  Metabase and Tableau. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Demonstrated \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " in building or delivering data store/warehouse solutions using ML  NLP  etc. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Nice to have:  Experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Python\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">SaaS or cloud-based app product \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Experience and success in both \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    small startup environments\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    large matrix organizations\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " working with large financial institutions. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " exploring new capabilities and technologies to drive innovation.   </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">We love to turn data into stories - stories about money falling through the cracks  success stories about our products and services  but most importantly stories about our customers and how we can enhance their \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have advanced working SQL knowledge and \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " working with relational databases  query authoring (SQL) as well as working familiarity with a variety of databases </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have minimum 4-7 years of \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " in a similar role and/or related education with demonstrated performance  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Python \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " Hive  Spark  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">NiFi experience B2B marketing Experience with cloud (GCP AWS or Azure) \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " with the \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Salesforce\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Einstein Data Science / Modeling \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " or working in a Data Science team  Contract would be until 31st Dec 2021 with a high possibility of extension.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Must-have:  4-6 years of experience with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    data engineering skills\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " 4-6 years of hands on experience with SQL  \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Shell Scripting\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Python\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">4-6 years of hands on experience ingesting data using APIs 4-6 years of hands on experience with BI tools / \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Big data technologies\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " like Hadoop  Spark  Scala  Nice-to-Have:  Knowledge of Dialogflow ES and CX APIs  BigQuery  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ":  data engineering: 4 years (required) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_extraction(ads[:10], extract_adp_conj_experience)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "fcd40894",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_verb_maybeadj_noun_experience(doc, label='EXPERIENCE'):\n",
    "    for tok in doc:\n",
    "        if tok.lower_ in EXP_TERMS:\n",
    "            for child in tok.rights:\n",
    "                if child.dep_ == 'acl':\n",
    "                    for gc in child.children:\n",
    "                        if gc.dep_ == 'prep':\n",
    "                            for ggc in gc.children:\n",
    "                                if ggc.dep_ == 'pobj':\n",
    "                                    for c in get_conjugations(ggc):\n",
    "                                        yield get_left_span(c, 'EXPERIENCE')\n",
    "                        elif gc.dep_ == 'dobj':\n",
    "                            for c in get_conjugations(gc):\n",
    "                                yield get_left_span(c, 'EXPERIENCE')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "bfde680b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">We help insurers fully automate more claims  deliver a great customer \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " while protecting against risk and accurately identifying suspected fraud  making internal teams more effective and improving financial performance.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Shift Technology offers a unique opportunity to brilliant candidates to accelerate their careers in data science  Data scientists work on a broad range of subjects  acquiring a lot of technical and professional \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " in data science  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">At the Intact Lab  we use machine learning  data science  software engineering  AI  agility  UX and design thinking to transform the customer \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " for millions of Canadians. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Your 8 years of \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " in the field of advanced statistics  data mining and text mining Strong communication  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">A multi-platform production \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " with commercial and open-source data mining frameworks like Python  R  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">They understand the technical aspects of data engineering and analysis but are focused on what the data is saying and how we can use it to provide a more personalized and targeted \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " for our customers.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Analyze data and developer feedback to understand needs  reduce time to onboard and build on the platform  and improve overall stakeholder \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">What we’re looking for:  Degree in Computer Science or equivalent with 5+ years of data analysis and product management \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Hands on data extraction and analytical \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " with SQL  Google Firebase  Metabase and Tableau. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Demonstrated \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " in building or delivering data store/warehouse solutions using ML  NLP  etc. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Nice to have:  \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " with Python. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">SaaS or cloud-based app product \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " and success in both small startup environments and large matrix organizations. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Experience working with \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    large financial institutions\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Experience exploring \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    new capabilities\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " and \n",
       "<mark class=\"entity\" style=\"background: lightgreen; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    technologies\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " to drive innovation.   </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">We love to turn data into stories - stories about money falling through the cracks  success stories about our products and services  but most importantly stories about our customers and how we can enhance their \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ". </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have advanced working SQL knowledge and \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " working with relational databases  query authoring (SQL) as well as working familiarity with a variety of databases </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">You have minimum 4-7 years of \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " in a similar role and/or related education with demonstrated performance  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Python \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " Hive  Spark  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">NiFi \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " B2B marketing Experience with cloud (GCP AWS or Azure) Experience with the Salesforce </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Einstein Data Science / Modeling \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " or working in a Data Science team  Contract would be until 31st Dec 2021 with a high possibility of extension.  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Must-have:  4-6 years of \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " with data engineering skills 4-6 years of hands on experience with SQL  Shell Scripting and Python </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">4-6 years of hands on \n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       " ingesting data using APIs 4-6 years of hands on experience with BI tools / Big data technologies like Hadoop  Spark  Scala  Nice-to-Have:  Knowledge of Dialogflow ES and CX APIs  BigQuery  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
       "<mark class=\"entity\" style=\"background: pink; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    Experience\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">MISSING</span>\n",
       "</mark>\n",
       ":  data engineering: 4 years (required) </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_extraction(ads[5:10], extract_verb_maybeadj_noun_experience)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "b710c8c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_exps = [extract_adp_conj_experience,]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "6c11a743",
   "metadata": {},
   "outputs": [],
   "source": [
    "n_ads = len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "5712756b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 36.3 s, sys: 1.81 s, total: 38.1 s\n",
      "Wall time: 38.2 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_ents = extract_df(*extract_exps, n_max=n_ads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "734bec55",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>docidx</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>label</th>\n",
       "      <th>sent_start</th>\n",
       "      <th>sent_end</th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Location</th>\n",
       "      <th>Job Board</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>application</td>\n",
       "      <td>0</td>\n",
       "      <td>242</td>\n",
       "      <td>243</td>\n",
       "      <td>EXPERIENCE</td>\n",
       "      <td>224</td>\n",
       "      <td>251</td>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Metricsflow Inc</td>\n",
       "      <td>At Metricsflow  we work to shape the future of...</td>\n",
       "      <td>St. John's</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>keras</td>\n",
       "      <td>1</td>\n",
       "      <td>162</td>\n",
       "      <td>163</td>\n",
       "      <td>EXPERIENCE</td>\n",
       "      <td>136</td>\n",
       "      <td>169</td>\n",
       "      <td>Data Scientist - 05/12/21</td>\n",
       "      <td>Acerta Analytics Solutions Inc</td>\n",
       "      <td>Acerta’s machine learning platforms leverage a...</td>\n",
       "      <td>Kitchener</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>operations data</td>\n",
       "      <td>1</td>\n",
       "      <td>175</td>\n",
       "      <td>177</td>\n",
       "      <td>EXPERIENCE</td>\n",
       "      <td>169</td>\n",
       "      <td>178</td>\n",
       "      <td>Data Scientist - 05/12/21</td>\n",
       "      <td>Acerta Analytics Solutions Inc</td>\n",
       "      <td>Acerta’s machine learning platforms leverage a...</td>\n",
       "      <td>Kitchener</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data</td>\n",
       "      <td>2</td>\n",
       "      <td>403</td>\n",
       "      <td>404</td>\n",
       "      <td>EXPERIENCE</td>\n",
       "      <td>396</td>\n",
       "      <td>413</td>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>GoodLife Fitness\\n</td>\n",
       "      <td>Position Description     DATA ENGINEER **This ...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cloud platforms</td>\n",
       "      <td>2</td>\n",
       "      <td>435</td>\n",
       "      <td>437</td>\n",
       "      <td>EXPERIENCE</td>\n",
       "      <td>413</td>\n",
       "      <td>481</td>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>GoodLife Fitness\\n</td>\n",
       "      <td>Position Description     DATA ENGINEER **This ...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2464</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>879</td>\n",
       "      <td>498</td>\n",
       "      <td>500</td>\n",
       "      <td>EXPERIENCE</td>\n",
       "      <td>485</td>\n",
       "      <td>563</td>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Cyient\\n</td>\n",
       "      <td>Cyient is a global engineering and technology ...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2465</th>\n",
       "      <td>cross-functional agile team</td>\n",
       "      <td>879</td>\n",
       "      <td>502</td>\n",
       "      <td>507</td>\n",
       "      <td>EXPERIENCE</td>\n",
       "      <td>485</td>\n",
       "      <td>563</td>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Cyient\\n</td>\n",
       "      <td>Cyient is a global engineering and technology ...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2466</th>\n",
       "      <td>NoSQL database technologies</td>\n",
       "      <td>880</td>\n",
       "      <td>523</td>\n",
       "      <td>526</td>\n",
       "      <td>EXPERIENCE</td>\n",
       "      <td>515</td>\n",
       "      <td>532</td>\n",
       "      <td>Principal Data Engineer</td>\n",
       "      <td>ELEMENTS Global Services</td>\n",
       "      <td>Elements Global Services is an award-winning H...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2467</th>\n",
       "      <td>modern programming languages</td>\n",
       "      <td>880</td>\n",
       "      <td>538</td>\n",
       "      <td>541</td>\n",
       "      <td>EXPERIENCE</td>\n",
       "      <td>532</td>\n",
       "      <td>549</td>\n",
       "      <td>Principal Data Engineer</td>\n",
       "      <td>ELEMENTS Global Services</td>\n",
       "      <td>Elements Global Services is an award-winning H...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2468</th>\n",
       "      <td>tools</td>\n",
       "      <td>880</td>\n",
       "      <td>542</td>\n",
       "      <td>543</td>\n",
       "      <td>EXPERIENCE</td>\n",
       "      <td>532</td>\n",
       "      <td>549</td>\n",
       "      <td>Principal Data Engineer</td>\n",
       "      <td>ELEMENTS Global Services</td>\n",
       "      <td>Elements Global Services is an award-winning H...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2469 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              text  docidx  start  end       label  \\\n",
       "0                      application       0    242  243  EXPERIENCE   \n",
       "1                            keras       1    162  163  EXPERIENCE   \n",
       "2                  operations data       1    175  177  EXPERIENCE   \n",
       "3                             data       2    403  404  EXPERIENCE   \n",
       "4                  Cloud platforms       2    435  437  EXPERIENCE   \n",
       "...                            ...     ...    ...  ...         ...   \n",
       "2464                 Data Engineer     879    498  500  EXPERIENCE   \n",
       "2465   cross-functional agile team     879    502  507  EXPERIENCE   \n",
       "2466   NoSQL database technologies     880    523  526  EXPERIENCE   \n",
       "2467  modern programming languages     880    538  541  EXPERIENCE   \n",
       "2468                         tools     880    542  543  EXPERIENCE   \n",
       "\n",
       "      sent_start  sent_end                  Job Title  \\\n",
       "0            224       251             Data Scientist   \n",
       "1            136       169  Data Scientist - 05/12/21   \n",
       "2            169       178  Data Scientist - 05/12/21   \n",
       "3            396       413              Data Engineer   \n",
       "4            413       481              Data Engineer   \n",
       "...          ...       ...                        ...   \n",
       "2464         485       563              Data Engineer   \n",
       "2465         485       563              Data Engineer   \n",
       "2466         515       532    Principal Data Engineer   \n",
       "2467         532       549    Principal Data Engineer   \n",
       "2468         532       549    Principal Data Engineer   \n",
       "\n",
       "                        Company Name  \\\n",
       "0                    Metricsflow Inc   \n",
       "1     Acerta Analytics Solutions Inc   \n",
       "2     Acerta Analytics Solutions Inc   \n",
       "3                 GoodLife Fitness\\n   \n",
       "4                 GoodLife Fitness\\n   \n",
       "...                              ...   \n",
       "2464                        Cyient\\n   \n",
       "2465                        Cyient\\n   \n",
       "2466        ELEMENTS Global Services   \n",
       "2467        ELEMENTS Global Services   \n",
       "2468        ELEMENTS Global Services   \n",
       "\n",
       "                                        Job Description         Location  \\\n",
       "0     At Metricsflow  we work to shape the future of...       St. John's   \n",
       "1     Acerta’s machine learning platforms leverage a...        Kitchener   \n",
       "2     Acerta’s machine learning platforms leverage a...        Kitchener   \n",
       "3     Position Description     DATA ENGINEER **This ...  Midtown Toronto   \n",
       "4     Position Description     DATA ENGINEER **This ...  Midtown Toronto   \n",
       "...                                                 ...              ...   \n",
       "2464  Cyient is a global engineering and technology ...         Montreal   \n",
       "2465  Cyient is a global engineering and technology ...         Montreal   \n",
       "2466  Elements Global Services is an award-winning H...          Toronto   \n",
       "2467  Elements Global Services is an award-winning H...          Toronto   \n",
       "2468  Elements Global Services is an award-winning H...          Toronto   \n",
       "\n",
       "      Job Board  \n",
       "0     Glassdoor  \n",
       "1        Indeed  \n",
       "2        Indeed  \n",
       "3     Glassdoor  \n",
       "4     Glassdoor  \n",
       "...         ...  \n",
       "2464  Glassdoor  \n",
       "2465  Glassdoor  \n",
       "2466     Indeed  \n",
       "2467     Indeed  \n",
       "2468     Indeed  \n",
       "\n",
       "[2469 rows x 12 columns]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "6b385ad9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Requirements:  3+ years in data scripting languages e.g Python Hands-on experience in the \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    application\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       " of predictive modeling  artificial neural networks  </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">Minimum of 3 - 5 years of data science experience using python Strong ML and statistics background 2+ years of software development experience Experience with \n",
       "<mark class=\"entity\" style=\"background: #ddd; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
       "    keras\n",
       "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-left: 0.5rem\">EXPERIENCE</span>\n",
       "</mark>\n",
       "  tensorflow  and sklearn. </div></span>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "showent_df(df_ents[:2])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "82c94155",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>n_company</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>375</th>\n",
       "      <td>Python</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>696</th>\n",
       "      <td>data science</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>652</th>\n",
       "      <td>data</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>407</th>\n",
       "      <td>SQL</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>301</th>\n",
       "      <td>Machine Learning</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1273</th>\n",
       "      <td>systems</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>212</th>\n",
       "      <td>Experience</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>752</th>\n",
       "      <td>development</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1298</th>\n",
       "      <td>tools</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>937</th>\n",
       "      <td>machine</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  text  n_company\n",
       "375             Python         56\n",
       "696       data science         27\n",
       "652               data         24\n",
       "407                SQL         23\n",
       "301   Machine Learning         16\n",
       "1273           systems         14\n",
       "212         Experience         14\n",
       "752        development         13\n",
       "1298             tools         13\n",
       "937            machine         12"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ent_agg = aggregate_df(df_ents)\n",
    "df_ent_agg.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "3ffcede7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from flashtext import KeywordProcessor\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "ba52b3fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "keyword_processor = KeywordProcessor(case_sensitive=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "b8a49779",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "144"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#selecting most popular\n",
    "skills = df_ent_agg.query('n_company >= 3').text\n",
    "len(skills)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "53a4a54c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for skill in skills:\n",
    "    keyword_processor.add_keyword(skill)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "642c4c2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "69a4a876",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "144"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "skills = list(\n",
    "(df_ent_agg\n",
    " .query('n_company >= 3')\n",
    ").text\n",
    ")\n",
    "len(skills)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "26175c76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python                             data science                       data\n",
      "SQL                                Machine Learning                   systems\n",
      "Experience                         development                        tools\n",
      "machine                            data analysis                      Data Scientist\n",
      "big data tools                     data visualization                 big data technologies\n",
      "AWS                                analysis                           NoSQL databases\n",
      "programming languages              NoSQL                              implementation\n",
      "customers                          machine learning                   data analytics\n",
      "Docker                             Spark                              relational databases\n",
      "field                              AI                                 software development\n",
      "one                                data pipeline                      design\n",
      "object function scripting languagesdata visualization tools           technologies\n",
      "solutions                          statistical analysis               experience\n",
      "Java                               projects                           data engineering\n",
      "relational SQL                     application                        workflow management tools\n",
      "Data Science                       years                              ability\n",
      "libraries                          R                                  ML\n",
      "building                           model development                  role\n",
      "python                             frameworks                         Big Data\n",
      "cloud services                     programming language               Tableau\n",
      "related field                      databases                          reporting\n",
      "Data Engineer                      more                               Microsoft Azure\n",
      "Deep Learning frameworks           degree                             management\n",
      "retail                             Hadoop                             ETL tools\n",
      "applications                       Apache Spark                       environments\n",
      "equivalent combination             modern programming languages       data analyst\n",
      "analytics                          Databricks                         Git\n",
      "any                                NLP                                methods\n",
      "Analytics                          artificial intelligence            Azure\n",
      "cloud                              analytical tools                   areas\n",
      "schema design                      quantitative analysis              classification\n",
      "infrastructure                     Proficiency                        programming\n",
      "similar role                       web development                    job\n",
      "series analysis                    Data Engineering                   fields\n",
      "project management                 business intelligence              cloud environments\n",
      "Natural Language Processing        financial industry                 services\n",
      "business requirements              AWS cloud services                 Apache Airflow\n",
      "natural language processing        Scala                              SQL Server\n",
      "data management                    Kubernetes                         automation\n",
      "common data science toolkits       future                             query optimization\n",
      "GCP                                predictive modeling                track record\n",
      "related business areas             unstructured data                  variety\n",
      "area                               biotechnology industry             container\n",
      "modern R packages                  modelling                          techniques\n",
      "Big Data technologies              BigQuery                           Linux\n",
      "teams                              Data Science tools                 models\n",
      "mining                             CPG                                data scripting languages\n",
      "expert knowledge                   structured problem                 AWS technologies\n"
     ]
    }
   ],
   "source": [
    "n_max=1000\n",
    "for a,b,c in zip(skills[:n_max:3],skills[1:n_max:3],skills[2:n_max:3]):\n",
    "     print('{:<35}{:<35}{:<}'.format(a,b,c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "cf2828e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_extractions_2(examples, *extractors):\n",
    "    # Could use context instead of enumerate\n",
    "    doc = nlp(examples, disable=['ner'])\n",
    "    for ent in filter_spans([Span(doc, start, end, label) for extractor in extractors for label, start, end in extractor(doc)]):\n",
    "        sent = ent.root.sent\n",
    "        yield ent.text\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "c7c0efbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_skills(examples, *extractors):\n",
    "    return list(get_extractions_2(examples, *extractors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "6e8a2e2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['MySQL', 'BigQuery Experience']\n"
     ]
    }
   ],
   "source": [
    "print(list_skills(df['Job Description'][10], extract_adp_conj_experience))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "cab690b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None <function extract_adp_conj_experience at 0x7fe828866700>\n"
     ]
    }
   ],
   "source": [
    "#for debugging visually\n",
    "print(show_extraction(df['Job Description'][10]), extract_adp_conj_experience)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "0214c42c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 48.9 s, sys: 236 ms, total: 49.1 s\n",
      "Wall time: 49.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#create a new column in df with the fxn that works\n",
    "df['skills'] = df['Job Description'].apply(lambda x: list_skills(x, extract_adp_conj_experience)) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "50634384",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Location</th>\n",
       "      <th>Job Board</th>\n",
       "      <th>skills</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Metricsflow Inc</td>\n",
       "      <td>At Metricsflow  we work to shape the future of...</td>\n",
       "      <td>St. John's</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[application]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Data Scientist - 05/12/21</td>\n",
       "      <td>Acerta Analytics Solutions Inc</td>\n",
       "      <td>Acerta’s machine learning platforms leverage a...</td>\n",
       "      <td>Kitchener</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[keras, operations data]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>GoodLife Fitness\\n</td>\n",
       "      <td>Position Description     DATA ENGINEER **This ...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[data, Cloud platforms, Azure Cloud Data Solut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Safety Aggregate Reporting &amp; Analytics Coordin...</td>\n",
       "      <td>IQVIA</td>\n",
       "      <td>Job Overview Manage the administrative support...</td>\n",
       "      <td>Kirkland</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Senior Data Engineer</td>\n",
       "      <td>Samsung Electronics\\n</td>\n",
       "      <td>Position Summary  The SmartThings Big Data tea...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[big data engineering, large scale, high avail...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           Job Title  \\\n",
       "0                                     Data Scientist   \n",
       "1                          Data Scientist - 05/12/21   \n",
       "2                                      Data Engineer   \n",
       "3  Safety Aggregate Reporting & Analytics Coordin...   \n",
       "4                               Senior Data Engineer   \n",
       "\n",
       "                     Company Name  \\\n",
       "0                 Metricsflow Inc   \n",
       "1  Acerta Analytics Solutions Inc   \n",
       "2              GoodLife Fitness\\n   \n",
       "3                           IQVIA   \n",
       "4           Samsung Electronics\\n   \n",
       "\n",
       "                                     Job Description         Location  \\\n",
       "0  At Metricsflow  we work to shape the future of...       St. John's   \n",
       "1  Acerta’s machine learning platforms leverage a...        Kitchener   \n",
       "2  Position Description     DATA ENGINEER **This ...  Midtown Toronto   \n",
       "3  Job Overview Manage the administrative support...         Kirkland   \n",
       "4  Position Summary  The SmartThings Big Data tea...        Vancouver   \n",
       "\n",
       "   Job Board                                             skills  \n",
       "0  Glassdoor                                      [application]  \n",
       "1     Indeed                           [keras, operations data]  \n",
       "2  Glassdoor  [data, Cloud platforms, Azure Cloud Data Solut...  \n",
       "3     Indeed                                                 []  \n",
       "4  Glassdoor  [big data engineering, large scale, high avail...  "
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "8697f187",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Job Title</th>\n",
       "      <th>Company Name</th>\n",
       "      <th>Job Description</th>\n",
       "      <th>Location</th>\n",
       "      <th>Job Board</th>\n",
       "      <th>skills</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>852</th>\n",
       "      <td>HEAD OF ANALYTICS &amp; INSIGHTS/DATA SCIENTIST</td>\n",
       "      <td>Saje Natural Wellness</td>\n",
       "      <td>For over 25 years we’ve been helping our commu...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[multivariate statistics, media measurement, c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>Machine Learning Engineer</td>\n",
       "      <td>Aviva</td>\n",
       "      <td>About the job Individually we are people  but ...</td>\n",
       "      <td>Markham</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[API deployment, microservice architecture, sk...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>627</th>\n",
       "      <td>Data Engineer / Power BI Developer, Omnia AI</td>\n",
       "      <td>Deloitte\\n</td>\n",
       "      <td>Job Type: Permanent Primary Location: Vancouve...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[high proficiency]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>Junior Business Intelligence Developer - Data ...</td>\n",
       "      <td>CAPRiser Group</td>\n",
       "      <td>Our subsidiary is in PropTech and real estate ...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[CSS, JavaScript, JavaScript, one, more, non-r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>661</th>\n",
       "      <td>Data Scientist Technical Lead (Montreal), Inta...</td>\n",
       "      <td>Intact\\n</td>\n",
       "      <td>Who needs insurance? Everybody. That keeps us ...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[field, source data mining frameworks]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>489</th>\n",
       "      <td>Bio informaticien/Bioinformatics scientist</td>\n",
       "      <td>GenAIz\\n</td>\n",
       "      <td>Avant le SRAS-CoV-2  un vaccin qui était dével...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[multidisciplinary bioinformatics projects Exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>434</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Awakedata</td>\n",
       "      <td>RESPONSIBILITIES: Develop state-of-the-art com...</td>\n",
       "      <td>Burnaby</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[big data techniques, relational, NoSQl Databa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>448</th>\n",
       "      <td>Scientifique de données / Data Scientist</td>\n",
       "      <td>BusPatrol\\n</td>\n",
       "      <td>Qui sommes-nous : BusPatrouille est une entrep...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>634</th>\n",
       "      <td>Data Management Specialist - Immunotoxicology</td>\n",
       "      <td>Charles River Laboratories\\n</td>\n",
       "      <td>For 70 years  Charles River employees have wor...</td>\n",
       "      <td>Senneville</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>748</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>F8th</td>\n",
       "      <td>We're transforming the Cyber Security industry...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[statistical package, software development, ma...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>831</th>\n",
       "      <td>Senior Professional Environmental Scientist (G...</td>\n",
       "      <td>Madrone Environmental Services Ltd</td>\n",
       "      <td>Senior Professional Environmental Scientist an...</td>\n",
       "      <td>Abbotsford</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[Western Canada]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Novisto</td>\n",
       "      <td>At Novisto  our vision is to be the world's le...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[new employee, cloud services, infrastructure,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>438</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Peritus.ai</td>\n",
       "      <td>About Peritus.ai   Peritus is dedicated to bui...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[products, Natural Language Processing, Deep L...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Junior Project Scientist / Junior Project Tech...</td>\n",
       "      <td>SNC-Lavalin\\n</td>\n",
       "      <td>Junior Project Scientist / Junior Project Tech...</td>\n",
       "      <td>Fort St. John</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Data Engineer - 313378</td>\n",
       "      <td>Procom\\n</td>\n",
       "      <td>Ingénieur de données  Dans le cadre de ses ent...</td>\n",
       "      <td>Montreal</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>383</th>\n",
       "      <td>Senior Data Analyst</td>\n",
       "      <td>ZTR</td>\n",
       "      <td>Are you passionate about implementing and main...</td>\n",
       "      <td>London</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[NoSQL databases, Python]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>591</th>\n",
       "      <td>Data Scientist - 05/12/21</td>\n",
       "      <td>Acerta Analytics Solutions Inc\\n</td>\n",
       "      <td>Acerta’s machine learning platforms leverage a...</td>\n",
       "      <td>Kitchener</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[keras, operations data]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>577</th>\n",
       "      <td>Senior Data Engineer - Remote</td>\n",
       "      <td>Koho\\n</td>\n",
       "      <td>About KOHO KOHO is a quickly scaling FinTech c...</td>\n",
       "      <td>Canada</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[testing, maintenance, Familiar, knowing, Kube...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>341</th>\n",
       "      <td>DATA SCIENTIST</td>\n",
       "      <td>Leger\\n</td>\n",
       "      <td>WHO WE ARE  At Leger  we know Canadians. With ...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[SQL, business intelligence, data, large datab...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>Data Science Manager, Omnia AI</td>\n",
       "      <td>Deloitte\\n</td>\n",
       "      <td>Job Type: Permanent Primary Location: Toronto ...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[Machine Learning]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>416</th>\n",
       "      <td>DATA ENGINEER</td>\n",
       "      <td>The Globe and Mail\\n</td>\n",
       "      <td>DATA ENGINEER POSITION CODE: 2021-063 LOCATION...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[technologies, big data tools, object function...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>352</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Mariner Partners\\n</td>\n",
       "      <td>Company Description As one of the largest prof...</td>\n",
       "      <td>Halifax</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[large enterprise]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>733</th>\n",
       "      <td>Principal Data Scientist</td>\n",
       "      <td>Veeva Systems</td>\n",
       "      <td>Veeva [NYSE: VEEV] is the leader in cloud-base...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[point person, projects]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>Data Engineer - Mississauga</td>\n",
       "      <td>Randstad\\n</td>\n",
       "      <td>Our client  located in Mississauga Ontario is ...</td>\n",
       "      <td>Mississauga</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[NoSQL systems Knowledge, testing systems]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>708</th>\n",
       "      <td>Product Development Confectionary Scientist</td>\n",
       "      <td>Cronos Group Inc.\\n</td>\n",
       "      <td>At Cronos Group  we hire talented people who t...</td>\n",
       "      <td>Stayner</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Machine Learning Scientist</td>\n",
       "      <td>Amazon Dev Centre Canada ULC\\n</td>\n",
       "      <td>Bachelor’s degree in Computer Science  Statist...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[machine, Python, extensive catalog]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>208</th>\n",
       "      <td>Data Engineering Intern (Fall 2021) - Toronto</td>\n",
       "      <td>Coursera\\n</td>\n",
       "      <td>Are you ready to join a team of tech-savvy mus...</td>\n",
       "      <td>Remote</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[Python, frameworks]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>537</th>\n",
       "      <td>Manager, Data Science - Research &amp; Analytics T...</td>\n",
       "      <td>Tonal\\n</td>\n",
       "      <td>Who We Are  Tonal is the smartest home gym and...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>639</th>\n",
       "      <td>Ingénieur de données senior / Sr. Data Engineer</td>\n",
       "      <td>McKesson Canada Corporation / La Corporation M...</td>\n",
       "      <td>English version to follow  McKesson touche la ...</td>\n",
       "      <td>Saint-Laurent</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[ETL Developer, scale enterprise data warehous...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>669</th>\n",
       "      <td>Solution Architect (Big Data) - Canada</td>\n",
       "      <td>Wavicle Data Solutions\\n</td>\n",
       "      <td>Wavicle Data Solutions designs and delivers da...</td>\n",
       "      <td>Remote</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[AWS technologies]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>Data Analyst Health Surveillance - BC Centre f...</td>\n",
       "      <td>PHSA\\n</td>\n",
       "      <td>Data Analyst Health Surveillance Epidemiology ...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[data analyst]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>Optimization Specialist, Analytics</td>\n",
       "      <td>Torstar</td>\n",
       "      <td>Company Description  Torstar Corporation is a ...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[media Development]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>761</th>\n",
       "      <td>Machine Learning Engineer / Data Scientist</td>\n",
       "      <td>TerraSense Analytics Ltd</td>\n",
       "      <td>CATCHY INTRO  Honestly  we’ve been brainstormi...</td>\n",
       "      <td>Kelowna</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[building, shipping, Docker, service]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>Data Engineer, Data Lake</td>\n",
       "      <td>Altus Group\\n</td>\n",
       "      <td>Job Category:  Software  Opportunity Awaits at...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>686</th>\n",
       "      <td>Product Manager - Data Science - Toronto Hub</td>\n",
       "      <td>Veeva Systems</td>\n",
       "      <td>Veeva [NYSE: VEEV] is the leader in cloud-base...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[prescription claims]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Oxford Properties</td>\n",
       "      <td>Why join us?  Are you looking to join an innov...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[data scientist, framework, plus, application ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>726</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>Pinterest\\n</td>\n",
       "      <td>About Pinterest:  Millions of people across th...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[big data, data warehouse]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>Cloud Data Engineer</td>\n",
       "      <td>Digital On Us\\n</td>\n",
       "      <td>Who is DigitalOnUs?  At DigitalOnUs  we not on...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[Familiarity, data, algorithms, SQL Server, Bi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>The Green Organic Dutchman Holdings Ltd</td>\n",
       "      <td>The Green Organic Dutchman is a global leader ...</td>\n",
       "      <td>Mississauga</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[field, Apache Airflow, Docker, web scraping, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>750</th>\n",
       "      <td>Principal Clinical Data Scientist Lead</td>\n",
       "      <td>PRA Health Sciences\\n</td>\n",
       "      <td>Do you want to watch clinical development chan...</td>\n",
       "      <td>Remote</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>Data Solution Architect</td>\n",
       "      <td>Onix Networking Corp\\n</td>\n",
       "      <td>Summary: Onix Solution Architect helps custome...</td>\n",
       "      <td>Quebec</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[infrastructure products, big data concepts, R...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>370</th>\n",
       "      <td>Data Scientist</td>\n",
       "      <td>Canadian Tire\\n</td>\n",
       "      <td>The CTC Personalization &amp; Customer Analytics t...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[business environment]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>582</th>\n",
       "      <td>Data Engineer/Analyst - Active Safety &amp; Autono...</td>\n",
       "      <td>General Motors\\n</td>\n",
       "      <td>About GM  There’s never been a more exciting t...</td>\n",
       "      <td>Ontario</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[Experience, vehicle data acquisition tools, B...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <td>Senior Data Scientist</td>\n",
       "      <td>Interac\\n</td>\n",
       "      <td>At Interac  we design products and solutions t...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[Data Science, Python, Financial industry, dat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>728</th>\n",
       "      <td>Applied Scientist - Machine Learning</td>\n",
       "      <td>Microsoft\\n</td>\n",
       "      <td>Applied Scientist - Machine Learning  The Inte...</td>\n",
       "      <td>Vancouver</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>673</th>\n",
       "      <td>Senior Machine Learning Engineer</td>\n",
       "      <td>Clearco</td>\n",
       "      <td>Clearco is a company built by founders for fou...</td>\n",
       "      <td>Ontario</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[founders, XGBoost deployments]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>664</th>\n",
       "      <td>Specialist, Loyalty Analytics</td>\n",
       "      <td>Cineplex</td>\n",
       "      <td>Company Description  Life at Cineplex: Cineple...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[data analytics, business intelligence, loyalty]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>Data Engineer</td>\n",
       "      <td>ScanSource Canada Inc.\\n</td>\n",
       "      <td>Summary:  The Data Engineer will be responsibl...</td>\n",
       "      <td>Mississauga</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[management solutions, Azure Data Factory, Dat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>Cloud Data Engineer</td>\n",
       "      <td>apptoza inc</td>\n",
       "      <td>Role: Cloud Data Engineer  Location: Toronto  ...</td>\n",
       "      <td>Midtown Toronto</td>\n",
       "      <td>Glassdoor</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>753</th>\n",
       "      <td>Senior Data Analyst, Growth</td>\n",
       "      <td>NorthOne</td>\n",
       "      <td>We make small businesses more successful throu...</td>\n",
       "      <td>Toronto</td>\n",
       "      <td>Indeed</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             Job Title  \\\n",
       "852        HEAD OF ANALYTICS & INSIGHTS/DATA SCIENTIST   \n",
       "207                          Machine Learning Engineer   \n",
       "627       Data Engineer / Power BI Developer, Omnia AI   \n",
       "671  Junior Business Intelligence Developer - Data ...   \n",
       "661  Data Scientist Technical Lead (Montreal), Inta...   \n",
       "489         Bio informaticien/Bioinformatics scientist   \n",
       "434                                     Data Scientist   \n",
       "448           Scientifique de données / Data Scientist   \n",
       "634      Data Management Specialist - Immunotoxicology   \n",
       "748                                     Data Scientist   \n",
       "831  Senior Professional Environmental Scientist (G...   \n",
       "396                                     Data Scientist   \n",
       "438                                     Data Scientist   \n",
       "11   Junior Project Scientist / Junior Project Tech...   \n",
       "59                              Data Engineer - 313378   \n",
       "383                                Senior Data Analyst   \n",
       "591                          Data Scientist - 05/12/21   \n",
       "577                      Senior Data Engineer - Remote   \n",
       "341                                     DATA SCIENTIST   \n",
       "355                     Data Science Manager, Omnia AI   \n",
       "416                                      DATA ENGINEER   \n",
       "352                                      Data Engineer   \n",
       "733                           Principal Data Scientist   \n",
       "456                        Data Engineer - Mississauga   \n",
       "708        Product Development Confectionary Scientist   \n",
       "12                          Machine Learning Scientist   \n",
       "208      Data Engineering Intern (Fall 2021) - Toronto   \n",
       "537  Manager, Data Science - Research & Analytics T...   \n",
       "639    Ingénieur de données senior / Sr. Data Engineer   \n",
       "669             Solution Architect (Big Data) - Canada   \n",
       "698  Data Analyst Health Surveillance - BC Centre f...   \n",
       "266                 Optimization Specialist, Analytics   \n",
       "761         Machine Learning Engineer / Data Scientist   \n",
       "399                           Data Engineer, Data Lake   \n",
       "686       Product Manager - Data Science - Toronto Hub   \n",
       "267                                     Data Scientist   \n",
       "726                                      Data Engineer   \n",
       "193                                Cloud Data Engineer   \n",
       "60                                       Data Engineer   \n",
       "750             Principal Clinical Data Scientist Lead   \n",
       "179                            Data Solution Architect   \n",
       "370                                     Data Scientist   \n",
       "582  Data Engineer/Analyst - Active Safety & Autono...   \n",
       "512                              Senior Data Scientist   \n",
       "728               Applied Scientist - Machine Learning   \n",
       "673                   Senior Machine Learning Engineer   \n",
       "664                      Specialist, Loyalty Analytics   \n",
       "176                                      Data Engineer   \n",
       "283                                Cloud Data Engineer   \n",
       "753                        Senior Data Analyst, Growth   \n",
       "\n",
       "                                          Company Name  \\\n",
       "852                              Saje Natural Wellness   \n",
       "207                                              Aviva   \n",
       "627                                         Deloitte\\n   \n",
       "671                                     CAPRiser Group   \n",
       "661                                           Intact\\n   \n",
       "489                                           GenAIz\\n   \n",
       "434                                          Awakedata   \n",
       "448                                        BusPatrol\\n   \n",
       "634                       Charles River Laboratories\\n   \n",
       "748                                               F8th   \n",
       "831                 Madrone Environmental Services Ltd   \n",
       "396                                            Novisto   \n",
       "438                                         Peritus.ai   \n",
       "11                                       SNC-Lavalin\\n   \n",
       "59                                            Procom\\n   \n",
       "383                                                ZTR   \n",
       "591                   Acerta Analytics Solutions Inc\\n   \n",
       "577                                             Koho\\n   \n",
       "341                                            Leger\\n   \n",
       "355                                         Deloitte\\n   \n",
       "416                               The Globe and Mail\\n   \n",
       "352                                 Mariner Partners\\n   \n",
       "733                                      Veeva Systems   \n",
       "456                                         Randstad\\n   \n",
       "708                                Cronos Group Inc.\\n   \n",
       "12                      Amazon Dev Centre Canada ULC\\n   \n",
       "208                                         Coursera\\n   \n",
       "537                                            Tonal\\n   \n",
       "639  McKesson Canada Corporation / La Corporation M...   \n",
       "669                           Wavicle Data Solutions\\n   \n",
       "698                                             PHSA\\n   \n",
       "266                                            Torstar   \n",
       "761                           TerraSense Analytics Ltd   \n",
       "399                                      Altus Group\\n   \n",
       "686                                      Veeva Systems   \n",
       "267                                  Oxford Properties   \n",
       "726                                        Pinterest\\n   \n",
       "193                                    Digital On Us\\n   \n",
       "60             The Green Organic Dutchman Holdings Ltd   \n",
       "750                              PRA Health Sciences\\n   \n",
       "179                             Onix Networking Corp\\n   \n",
       "370                                    Canadian Tire\\n   \n",
       "582                                   General Motors\\n   \n",
       "512                                          Interac\\n   \n",
       "728                                        Microsoft\\n   \n",
       "673                                            Clearco   \n",
       "664                                           Cineplex   \n",
       "176                           ScanSource Canada Inc.\\n   \n",
       "283                                        apptoza inc   \n",
       "753                                           NorthOne   \n",
       "\n",
       "                                       Job Description         Location  \\\n",
       "852  For over 25 years we’ve been helping our commu...        Vancouver   \n",
       "207  About the job Individually we are people  but ...          Markham   \n",
       "627  Job Type: Permanent Primary Location: Vancouve...        Vancouver   \n",
       "671  Our subsidiary is in PropTech and real estate ...          Toronto   \n",
       "661  Who needs insurance? Everybody. That keeps us ...         Montreal   \n",
       "489  Avant le SRAS-CoV-2  un vaccin qui était dével...         Montreal   \n",
       "434  RESPONSIBILITIES: Develop state-of-the-art com...          Burnaby   \n",
       "448  Qui sommes-nous : BusPatrouille est une entrep...         Montreal   \n",
       "634  For 70 years  Charles River employees have wor...       Senneville   \n",
       "748  We're transforming the Cyber Security industry...  Midtown Toronto   \n",
       "831  Senior Professional Environmental Scientist an...       Abbotsford   \n",
       "396  At Novisto  our vision is to be the world's le...         Montreal   \n",
       "438  About Peritus.ai   Peritus is dedicated to bui...         Montreal   \n",
       "11   Junior Project Scientist / Junior Project Tech...    Fort St. John   \n",
       "59   Ingénieur de données  Dans le cadre de ses ent...         Montreal   \n",
       "383  Are you passionate about implementing and main...           London   \n",
       "591  Acerta’s machine learning platforms leverage a...        Kitchener   \n",
       "577  About KOHO KOHO is a quickly scaling FinTech c...           Canada   \n",
       "341  WHO WE ARE  At Leger  we know Canadians. With ...  Midtown Toronto   \n",
       "355  Job Type: Permanent Primary Location: Toronto ...  Midtown Toronto   \n",
       "416  DATA ENGINEER POSITION CODE: 2021-063 LOCATION...  Midtown Toronto   \n",
       "352  Company Description As one of the largest prof...          Halifax   \n",
       "733  Veeva [NYSE: VEEV] is the leader in cloud-base...          Toronto   \n",
       "456  Our client  located in Mississauga Ontario is ...      Mississauga   \n",
       "708  At Cronos Group  we hire talented people who t...          Stayner   \n",
       "12   Bachelor’s degree in Computer Science  Statist...        Vancouver   \n",
       "208  Are you ready to join a team of tech-savvy mus...           Remote   \n",
       "537  Who We Are  Tonal is the smartest home gym and...  Midtown Toronto   \n",
       "639  English version to follow  McKesson touche la ...    Saint-Laurent   \n",
       "669  Wavicle Data Solutions designs and delivers da...           Remote   \n",
       "698  Data Analyst Health Surveillance Epidemiology ...        Vancouver   \n",
       "266  Company Description  Torstar Corporation is a ...          Toronto   \n",
       "761  CATCHY INTRO  Honestly  we’ve been brainstormi...          Kelowna   \n",
       "399  Job Category:  Software  Opportunity Awaits at...  Midtown Toronto   \n",
       "686  Veeva [NYSE: VEEV] is the leader in cloud-base...          Toronto   \n",
       "267  Why join us?  Are you looking to join an innov...          Toronto   \n",
       "726  About Pinterest:  Millions of people across th...  Midtown Toronto   \n",
       "193  Who is DigitalOnUs?  At DigitalOnUs  we not on...  Midtown Toronto   \n",
       "60   The Green Organic Dutchman is a global leader ...      Mississauga   \n",
       "750  Do you want to watch clinical development chan...           Remote   \n",
       "179  Summary: Onix Solution Architect helps custome...           Quebec   \n",
       "370  The CTC Personalization & Customer Analytics t...  Midtown Toronto   \n",
       "582  About GM  There’s never been a more exciting t...          Ontario   \n",
       "512  At Interac  we design products and solutions t...  Midtown Toronto   \n",
       "728  Applied Scientist - Machine Learning  The Inte...        Vancouver   \n",
       "673  Clearco is a company built by founders for fou...          Ontario   \n",
       "664  Company Description  Life at Cineplex: Cineple...          Toronto   \n",
       "176  Summary:  The Data Engineer will be responsibl...      Mississauga   \n",
       "283  Role: Cloud Data Engineer  Location: Toronto  ...  Midtown Toronto   \n",
       "753  We make small businesses more successful throu...          Toronto   \n",
       "\n",
       "     Job Board                                             skills  \n",
       "852     Indeed  [multivariate statistics, media measurement, c...  \n",
       "207     Indeed  [API deployment, microservice architecture, sk...  \n",
       "627  Glassdoor                                 [high proficiency]  \n",
       "671     Indeed  [CSS, JavaScript, JavaScript, one, more, non-r...  \n",
       "661  Glassdoor             [field, source data mining frameworks]  \n",
       "489  Glassdoor  [multidisciplinary bioinformatics projects Exp...  \n",
       "434  Glassdoor  [big data techniques, relational, NoSQl Databa...  \n",
       "448  Glassdoor                                                 []  \n",
       "634  Glassdoor                                                 []  \n",
       "748  Glassdoor  [statistical package, software development, ma...  \n",
       "831  Glassdoor                                   [Western Canada]  \n",
       "396  Glassdoor  [new employee, cloud services, infrastructure,...  \n",
       "438  Glassdoor  [products, Natural Language Processing, Deep L...  \n",
       "11   Glassdoor                                                 []  \n",
       "59   Glassdoor                                                 []  \n",
       "383     Indeed                          [NoSQL databases, Python]  \n",
       "591  Glassdoor                           [keras, operations data]  \n",
       "577  Glassdoor  [testing, maintenance, Familiar, knowing, Kube...  \n",
       "341  Glassdoor  [SQL, business intelligence, data, large datab...  \n",
       "355  Glassdoor                                 [Machine Learning]  \n",
       "416  Glassdoor  [technologies, big data tools, object function...  \n",
       "352  Glassdoor                                 [large enterprise]  \n",
       "733     Indeed                           [point person, projects]  \n",
       "456  Glassdoor         [NoSQL systems Knowledge, testing systems]  \n",
       "708  Glassdoor                                                 []  \n",
       "12   Glassdoor               [machine, Python, extensive catalog]  \n",
       "208  Glassdoor                               [Python, frameworks]  \n",
       "537  Glassdoor                                                 []  \n",
       "639  Glassdoor  [ETL Developer, scale enterprise data warehous...  \n",
       "669  Glassdoor                                 [AWS technologies]  \n",
       "698  Glassdoor                                     [data analyst]  \n",
       "266     Indeed                                [media Development]  \n",
       "761  Glassdoor              [building, shipping, Docker, service]  \n",
       "399  Glassdoor                                                 []  \n",
       "686     Indeed                              [prescription claims]  \n",
       "267     Indeed  [data scientist, framework, plus, application ...  \n",
       "726  Glassdoor                         [big data, data warehouse]  \n",
       "193  Glassdoor  [Familiarity, data, algorithms, SQL Server, Bi...  \n",
       "60   Glassdoor  [field, Apache Airflow, Docker, web scraping, ...  \n",
       "750  Glassdoor                                                 []  \n",
       "179  Glassdoor  [infrastructure products, big data concepts, R...  \n",
       "370  Glassdoor                             [business environment]  \n",
       "582  Glassdoor  [Experience, vehicle data acquisition tools, B...  \n",
       "512  Glassdoor  [Data Science, Python, Financial industry, dat...  \n",
       "728  Glassdoor                                                 []  \n",
       "673     Indeed                    [founders, XGBoost deployments]  \n",
       "664     Indeed   [data analytics, business intelligence, loyalty]  \n",
       "176  Glassdoor  [management solutions, Azure Data Factory, Dat...  \n",
       "283  Glassdoor                                                 []  \n",
       "753     Indeed                                                 []  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(50, random_state = 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "bf3c2eff",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('outputs/raw_skills.txt', 'w') as f:\n",
    "    for skill in skills:\n",
    "        print(skill, file=f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "6dd5e41b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('outputs/rule_based_extracted_skills.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f21c4ca",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}